From 16a7e0ce8fe83402b4ed00d8b02ae2475f6cd906 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 26 Nov 2025 17:01:23 -0300 Subject: [PATCH 01/32] fix: filter empty SSE lines in BaseModelResponseIterator to prevent extra empty chunks When streaming with stream_options={"include_usage": True}, xAI and other providers using BaseLLMHTTPHandler were returning an extra empty chunk after the usage chunk. This was caused by empty SSE lines (separators between events) being processed as empty GenericStreamingChunks. The fix adds a loop in __next__ and __anext__ to skip empty lines before processing, ensuring only meaningful SSE data events are converted to chunks. Fixes #17136 --- litellm/llms/base_llm/base_model_iterator.py | 89 +++++++++++--------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index 6953b1c587..62cd503a89 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -125,26 +125,32 @@ class BaseModelResponseIterator: ) def __next__(self): - try: - chunk = self.response_iterator.__next__() - except StopIteration: - raise StopIteration - except ValueError as e: - raise RuntimeError(f"Error receiving chunk from stream: {e}") + while True: + try: + chunk = self.response_iterator.__next__() + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") - try: - str_line = chunk - if isinstance(chunk, bytes): # Handle binary data - str_line = chunk.decode("utf-8") # Convert bytes to string - index = str_line.find("data:") - if index != -1: - str_line = str_line[index:] - # chunk is a str at this point - return self._handle_string_chunk(str_line=str_line) - except StopIteration: - raise StopIteration - except ValueError as e: - raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + try: + str_line = chunk + if isinstance(chunk, bytes): # Handle binary data + str_line = chunk.decode("utf-8") # Convert bytes to string + index = str_line.find("data:") + if index != -1: + str_line = str_line[index:] + + # Skip empty lines (common in SSE streams between events) + if not str_line or not str_line.strip(): + continue + + # chunk is a str at this point + return self._handle_string_chunk(str_line=str_line) + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") # Async iterator def __aiter__(self): @@ -152,30 +158,35 @@ class BaseModelResponseIterator: return self async def __anext__(self): - try: - chunk = await self.async_response_iterator.__anext__() + while True: + try: + chunk = await self.async_response_iterator.__anext__() - except StopAsyncIteration: - raise StopAsyncIteration - except ValueError as e: - raise RuntimeError(f"Error receiving chunk from stream: {e}") + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") - try: - str_line = chunk - if isinstance(chunk, bytes): # Handle binary data - str_line = chunk.decode("utf-8") # Convert bytes to string - index = str_line.find("data:") - if index != -1: - str_line = str_line[index:] + try: + str_line = chunk + if isinstance(chunk, bytes): # Handle binary data + str_line = chunk.decode("utf-8") # Convert bytes to string + index = str_line.find("data:") + if index != -1: + str_line = str_line[index:] - # chunk is a str at this point - chunk = self._handle_string_chunk(str_line=str_line) + # Skip empty lines (common in SSE streams between events) + if not str_line or not str_line.strip(): + continue - return chunk - except StopAsyncIteration: - raise StopAsyncIteration - except ValueError as e: - raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + # chunk is a str at this point + chunk = self._handle_string_chunk(str_line=str_line) + + return chunk + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") class MockResponseIterator: # for returning ai21 streaming responses From 8c128edb5d3790096376c086f9fa1027f6344e08 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 26 Nov 2025 17:05:40 -0300 Subject: [PATCH 02/32] test: add unit tests for BaseModelResponseIterator empty SSE line filtering Tests verify that empty lines between SSE events are properly filtered and don't produce extra empty chunks in streaming responses. --- .../llms/base_llm/test_base_model_iterator.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/test_litellm/llms/base_llm/test_base_model_iterator.py diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py new file mode 100644 index 0000000000..d5166c4690 --- /dev/null +++ b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py @@ -0,0 +1,117 @@ +""" +Tests for BaseModelResponseIterator - specifically testing that empty SSE lines are filtered +""" + +import pytest +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.types.utils import GenericStreamingChunk, ModelResponseStream + + +class TestBaseModelResponseIterator: + """Test cases for BaseModelResponseIterator empty line filtering""" + + def test_filter_empty_sse_lines_sync(self): + """ + Test that empty SSE lines (common between SSE events) are filtered out + and don't produce empty chunks. + + This fixes the bug where providers using BaseLLMHTTPHandler (like xAI) + would return extra empty chunks when streaming with include_usage=True. + + Related: GitHub Issue #17136 + """ + # Simulate SSE stream with empty lines between events (normal SSE format) + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hello"}}]}', + '', # Empty line (SSE separator) + 'data: {"id":"1","choices":[{"delta":{"content":" World"}}]}', + '', # Empty line (SSE separator) + 'data: {"id":"1","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5}}', + '', # Empty line (SSE separator) + 'data: [DONE]', + '', # Empty line after DONE + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # Should have 4 chunks: 2 content + 1 usage + 1 DONE + # Empty lines should be filtered out + assert len(chunks) == 4, f"Expected 4 chunks, got {len(chunks)}" + + # Verify no empty/None chunks were included + # The base iterator returns ModelResponseStream objects + for i, chunk in enumerate(chunks): + assert chunk is not None, f"Chunk {i} should not be None" + + def test_filter_whitespace_only_lines_sync(self): + """Test that lines with only whitespace are also filtered""" + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hi"}}]}', + ' ', # Whitespace only + '\t', # Tab only + 'data: [DONE]', + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # Should have 2 chunks: 1 content + 1 DONE + assert len(chunks) == 2, f"Expected 2 chunks, got {len(chunks)}" + + def test_valid_chunks_not_filtered_sync(self): + """Test that valid data chunks are not filtered""" + sse_lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}', + 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}', + 'data: {"id":"1","choices":[{"delta":{"content":"C"}}]}', + 'data: [DONE]', + ] + + iterator = BaseModelResponseIterator( + streaming_response=iter(sse_lines), + sync_stream=True + ) + + chunks = list(iterator) + + # All 4 chunks should be present + assert len(chunks) == 4, f"Expected 4 chunks, got {len(chunks)}" + + +@pytest.mark.asyncio +async def test_filter_empty_sse_lines_async(): + """ + Test async version: empty SSE lines should be filtered out + """ + async def async_sse_generator(): + lines = [ + 'data: {"id":"1","choices":[{"delta":{"content":"Hello"}}]}', + '', # Empty line + 'data: {"id":"1","choices":[{"delta":{"content":" World"}}]}', + '', # Empty line + 'data: [DONE]', + '', # Empty line + ] + for line in lines: + yield line + + iterator = BaseModelResponseIterator( + streaming_response=async_sse_generator(), + sync_stream=False + ) + + chunks = [] + async for chunk in iterator: + chunks.append(chunk) + + # Should have 3 chunks: 2 content + 1 DONE + assert len(chunks) == 3, f"Expected 3 chunks, got {len(chunks)}" From 265a08823c833858be10386c271107719c9aadbc Mon Sep 17 00:00:00 2001 From: Peter Chanthamynavong Date: Tue, 9 Dec 2025 08:00:07 -0800 Subject: [PATCH 03/32] refactor(files): add type aliases for provider parameters Introduces 5 type aliases for provider Literal types in the Files API: - FileCreateProvider, FileRetrieveProvider, FileDeleteProvider - FileListProvider, FileContentProvider Updates 10 function signatures to use the new aliases. Reduces duplication and improves readability. Closes #17608 --- litellm/files/main.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index acf545e431..b66096b013 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -13,6 +13,13 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast import httpx +# Type aliases for provider parameters +FileCreateProvider = Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] +FileRetrieveProvider = Literal["openai", "azure", "hosted_vllm"] +FileDeleteProvider = Literal["openai", "azure"] +FileListProvider = Literal["openai", "azure"] +FileContentProvider = Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] + import litellm from litellm import get_secret_str from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider @@ -58,7 +65,7 @@ anthropic_files_instance = AnthropicFilesHandler() async def acreate_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai", + custom_llm_provider: FileCreateProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -102,7 +109,7 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None, + custom_llm_provider: Optional[FileCreateProvider] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -281,7 +288,7 @@ def create_file( @client async def afile_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", + custom_llm_provider: FileRetrieveProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -322,7 +329,7 @@ async def afile_retrieve( @client def file_retrieve( file_id: str, - custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai", + custom_llm_provider: FileRetrieveProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -438,7 +445,7 @@ def file_retrieve( @client async def afile_delete( file_id: str, - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileDeleteProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -482,7 +489,7 @@ async def afile_delete( def file_delete( file_id: str, model: Optional[str] = None, - custom_llm_provider: Union[Literal["openai", "azure"], str] = "openai", + custom_llm_provider: Union[FileDeleteProvider, str] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -604,7 +611,7 @@ def file_delete( # List files @client async def afile_list( - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileListProvider = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -645,7 +652,7 @@ async def afile_list( @client def file_list( - custom_llm_provider: Literal["openai", "azure"] = "openai", + custom_llm_provider: FileListProvider = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -759,7 +766,7 @@ def file_list( @client async def afile_content( file_id: str, - custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai", + custom_llm_provider: FileContentProvider = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -803,9 +810,7 @@ async def afile_content( def file_content( file_id: str, model: Optional[str] = None, - custom_llm_provider: Optional[ - Union[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"], str] - ] = None, + custom_llm_provider: Optional[Union[FileContentProvider, str]] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, From 68ba9a6a99eac3ea91012fec313edbddf12cf6e5 Mon Sep 17 00:00:00 2001 From: Chesars Date: Mon, 5 Jan 2026 10:29:55 -0300 Subject: [PATCH 04/32] fix: enforce Black formatting in CI instead of auto-formatting Changed CI workflow to use `black --check` instead of `black .` This makes the CI fail if code is not formatted, rather than auto-formatting and discarding changes. Aligns with README.md promise that "all checks must pass" and follows Black best practices for CI/CD pipelines. --- .github/workflows/test-linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 35ebffeada..26f8a2efb6 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -34,10 +34,10 @@ jobs: poetry install --with dev poetry run pip install openai==1.100.1 - - name: Run Black formatting + - name: Check Black formatting run: | cd litellm - poetry run black . + poetry run black --check . cd .. - name: Debug - Check file state From a2f3beb26f15c7354257b2d4a84bf621a819f4f6 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 3 Mar 2026 21:47:55 -0300 Subject: [PATCH 05/32] Update tests/test_litellm/llms/base_llm/test_base_model_iterator.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/llms/base_llm/test_base_model_iterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py index d5166c4690..96cd299b2b 100644 --- a/tests/test_litellm/llms/base_llm/test_base_model_iterator.py +++ b/tests/test_litellm/llms/base_llm/test_base_model_iterator.py @@ -4,7 +4,7 @@ Tests for BaseModelResponseIterator - specifically testing that empty SSE lines import pytest from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.types.utils import GenericStreamingChunk, ModelResponseStream +from litellm.types.utils import GenericStreamingChunk class TestBaseModelResponseIterator: From a78bd9a468dfa064eb69bcd47e3e61800920d23b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:13:33 +0530 Subject: [PATCH 06/32] build(deps): bump hono from 4.10.6 to 4.12.7 in /litellm-js/spend-logs (#23312) * Rename 'Team-Based Guardrails' to 'Team Bring-Your-Own Guardrails' (#23307) Co-authored-by: Cursor Agent * build(deps): bump hono from 4.10.6 to 4.12.7 in /litellm-js/spend-logs Bumps [hono](https://github.com/honojs/hono) from 4.10.6 to 4.12.7. - [Release notes](https://github.com/honojs/hono/releases) - [Commits](https://github.com/honojs/hono/compare/v4.10.6...v4.12.7) --- updated-dependencies: - dependency-name: hono dependency-version: 4.12.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: Krish Dholakia Co-authored-by: Cursor Agent Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../docs/proxy/guardrails/team_based_guardrails.md | 2 +- docs/my-website/release_notes/v1.81.9.md | 2 +- litellm-js/spend-logs/package-lock.json | 8 ++++---- litellm-js/spend-logs/package.json | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/team_based_guardrails.md b/docs/my-website/docs/proxy/guardrails/team_based_guardrails.md index 2d55294a71..0e610b6e44 100644 --- a/docs/my-website/docs/proxy/guardrails/team_based_guardrails.md +++ b/docs/my-website/docs/proxy/guardrails/team_based_guardrails.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Team-Based Guardrails +# Team Bring-Your-Own Guardrails Team-based guardrails let **developers** register a guardrail for their team via the API; an **admin** then reviews and approves or rejects it in the LiteLLM UI. Only [Generic Guardrail API](/docs/adding_provider/generic_guardrail_api) guardrails can be registered this way. diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9.md index c7659442c4..80be4179b4 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9.md @@ -279,7 +279,7 @@ Let's dive in. - Add HTTP support to custom code guardrails + Unified guardrails for MCP + Agent guardrail support - [PR #20619](https://github.com/BerriAI/litellm/pull/20619) - Custom Code Guardrails UI Playground - [PR #20377](https://github.com/BerriAI/litellm/pull/20377) -- **Team-Based Guardrails** +- **Team Bring-Your-Own Guardrails** - Implement team-based isolation guardrails management - [PR #20318](https://github.com/BerriAI/litellm/pull/20318) - **[OpenAI Moderations](../../docs/apply_guardrail)** diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json index 1a13a76820..b24ff0a494 100644 --- a/litellm-js/spend-logs/package-lock.json +++ b/litellm-js/spend-logs/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "@hono/node-server": "^1.10.1", - "hono": "^4.10.3" + "hono": "^4.12.7" }, "devDependencies": { "@types/node": "^20.11.17", @@ -548,9 +548,9 @@ } }, "node_modules/hono": { - "version": "4.10.6", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.10.6.tgz", - "integrity": "sha512-BIdolzGpDO9MQ4nu3AUuDwHZZ+KViNm+EZ75Ae55eMXMqLVhDFqEMXxtUe9Qh8hjL+pIna/frs2j6Y2yD5Ua/g==", + "version": "4.12.7", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.7.tgz", + "integrity": "sha512-jq9l1DM0zVIvsm3lv9Nw9nlJnMNPOcAtsbsgiUhWcFzPE99Gvo6yRTlszSLLYacMeQ6quHD6hMfId8crVHvexw==", "license": "MIT", "engines": { "node": ">=16.9.0" diff --git a/litellm-js/spend-logs/package.json b/litellm-js/spend-logs/package.json index adfe49017d..a40b0fc2a8 100644 --- a/litellm-js/spend-logs/package.json +++ b/litellm-js/spend-logs/package.json @@ -4,7 +4,7 @@ }, "dependencies": { "@hono/node-server": "^1.10.1", - "hono": "^4.10.3" + "hono": "^4.12.7" }, "devDependencies": { "@types/node": "^20.11.17", From 3f18cd2fdc43e74eac3cdf750bbfe1a6f3ab02f2 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello <3691490+PeterDaveHello@users.noreply.github.com> Date: Wed, 11 Mar 2026 22:47:41 +0800 Subject: [PATCH 07/32] [Docs] Fix "Page Not Found" link for Anthropic endpoint (#23349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(anthropic): enforce type:'object' on tool input schemas Anthropic's API requires all tool input_schema to have type:'object' at the root level. When OpenAI-format tools have parameters with a missing or non-'object' type field (common with MCP tool servers), the schema was passed through unchanged, causing Anthropic to reject with: 'tools.N.custom.input_schema.type: Input should be object'. The existing default handles the case where parameters is entirely missing, but does not normalize schemas that ARE provided with a wrong or absent type field. Fix: After extracting _input_schema in _map_tool_helper(), ensure type is set to 'object' and properties exists. This matches the normalization already done implicitly by the Bedrock handler. Added 4 unit tests covering: missing type, wrong type, valid schema (no-op), and entirely missing parameters. Related issues: #12020, #64, #1671 * fix(anthropic): deduplicate tool_result messages by tool_call_id Anthropic requires exactly one tool_result per tool_use. When conversation history (e.g. from session resume/checkpoint restore) contains duplicate tool result messages with the same tool_call_id, the API rejects with: 'each tool_use must have a single result. Found multiple tool_result blocks with id: '. This is already handled for Bedrock via _deduplicate_bedrock_tool_content() but was missing from the Anthropic direct and Vertex AI partner paths, which share sanitize_messages_for_tool_calling(). Fix: Add Case D to sanitize_messages_for_tool_calling() — after the existing orphan detection passes, scan for duplicate tool_call_ids and keep only the last occurrence (most complete result). Added 3 unit tests: dedup with duplicates, no-op with unique IDs, and behavior when modify_params=False. Related issues: #11804, #11029, #6836, #1782, #151 * fix: shallow copy input_schema to avoid caller mutation + add mutation guard test Addresses Greptile review: - dict(_input_schema) before mutation prevents cross-provider state leakage - Test asserts original tool parameters dict is unchanged after call * feat: add qwen3.5 series for openrouter * fix: typo on max_output_tokens and max_tokens from qwen3.5 series * chore: fix * chore: fix * [Test] UI - Logs: Add unit tests for 5 untested view_logs components Add vitest tests for TypeBadges, ErrorViewer, ConfigInfoMessage, TimeCell, and TruncatedValue covering rendering, user interactions, and edge cases. Co-Authored-By: Claude Opus 4.6 * Rename 'Team-Based Guardrails' to 'Team Bring-Your-Own Guardrails' (#23307) Co-authored-by: Cursor Agent * feat(chat-ui): responses API + MCP tool execution in /chat (#23297) * feat(ui): add Chat UI v0 — standalone LiteLLM-branded chat window Adds a full chat UI accessible from the sidebar Chat link (opens in new tab). - Standalone route at /chat (outside dashboard layout — no Navbar/Sidebar chrome) - Claude.ai-style layout: model selector top-left, LiteLLM logo center, settings top-right - Greeting with time-of-day, centered input card, suggestion chips (Write/Learn/Code/Brainstorm) - Sliding conversation history sidebar with Cmd+K search, rename, delete, date grouping - localStorage-backed conversation persistence (litellm_chat_history_v1) - Streaming completions via makeOpenAIChatCompletionRequest with AbortController stop support - MCP server picker (toggle servers on/off per conversation) - LiteLLM aesthetic: white/light-gray background, Ant Design blue (#1677ff) primary, system font - Sidebar2: Chat menu item opens in new tab via window.open * feat(chat-ui): responses API + MCP tool execution display - Switch /chat from chat completions to responses API (previous_response_id session chaining) - Add MCP server picker with search filter in chat input bar - Show MCP tool call events (list_tools + call_tool) inline in chat via MCPEventsDisplay - Add tool chip strip showing available tools when MCP servers are selected - Non-blocking MCP toggle: server added immediately, verification in background (works for no-auth MCPs like deepwiki) - Add truncateAfterMessage to useChatHistory for edit/retry - Sync activeConversationId on URL change (fixes stale conversation on new chat) - Add "Open Chat" shortcut button to sidebar * fix(chat-ui): switch to responses API, remove dead code, add tests - Switch handleSend from makeOpenAIChatCompletionRequest to makeOpenAIResponsesRequest with previous_response_id session chaining - Add responsesSessionId state; reset to null when starting a new conversation - Remove unused ChatInputBar.tsx and ModelSelector.tsx (dead code) - Add tests/test_litellm/test_chat_ui_responses_session.py covering previous_response_id forwarding and signature validation * fix(chat-ui): address greptile review issues - Reset responsesSessionId when activeConversationId changes (not just on new conversation) - Wire onMCPEvent callback into makeOpenAIResponsesRequest; render MCPEventsDisplay below messages - Clear mcpEvents on each new send - Explicitly filter history to user/assistant roles only (no tool-role casting) - Remove duplicate "Chat" menu item from sidebar (pinned button serves same purpose) - Make Sider a flex column so "Open Chat" button actually pins to bottom - Fix tests to intercept real HTTP requests and assert previous_response_id in body * fix(chat-ui): address greptile review feedback (greploop iteration 1) - Fix duplicate context: when responsesSessionId is set, only send the new user message as input (prior context is already server-side via session chaining). Full history is still sent on the first turn. - Fix ephemeral MCP events: store events per-message in ChatMessage.mcpEvents instead of ephemeral component state. Events now survive across turns and render inline below each assistant response via MCPEventsDisplay. - Remove stale mcpEvents useState and ephemeral panel at bottom of chat. * fix(chat-ui): address greptile review feedback (greploop iteration 2) - Fix stale session on edit/retry: derive previousResponseId as null when historyOverride is set so edit/retry always starts a fresh Responses API session rather than chaining off a now-invalid prior session - Fix unsafe MCPEvent cast: import MCPEvent directly from MCPEventsDisplay into types.ts and type ChatMessage.mcpEvents as MCPEvent[], eliminating the bare 'as MCPEvent[]' cast in ChatMessages.tsx * fix(chat-ui): fix MCPEvent layering, batch localStorage writes, module-level test imports - Move MCPEvent interface definition into chat/types.ts (single source of truth) - MCPEventsDisplay.tsx now imports MCPEvent from types.ts instead of defining it locally - Batch MCP event localStorage writes: accumulate during stream, persist once in finally - Move test imports to module level per PEP 8 convention * fix(chat-ui): fix MCPEvent import path and rename truncateFromMessage - responses_api.tsx now imports MCPEvent directly from chat/types (not via MCPEventsDisplay re-export) - Remove the now-unnecessary MCPEvent re-export from MCPEventsDisplay.tsx - Rename truncateAfterMessage → truncateFromMessage: the function removes the target message and all subsequent ones (not just what comes after), so the new name accurately describes the behavior * fix(responses-api): fix whitespace token filter and MCP server URL construction - Drop the delta.trim() whitespace filter that was silently swallowing spaces and newlines during streaming, causing words to concatenate and paragraphs to collapse. Only skip truly empty strings (delta.length > 0). - Use proxyBaseUrl for MCP server_url construction instead of the hardcoded relative path "litellm_proxy/mcp", so non-root deployments route correctly. * fix(responses-api): use unique server_label per MCP server to prevent tool routing collisions * fix(chat-ui): move MCPEvent to shared mcp_tools/types, skip partial events on abort - Move MCPEvent interface to mcp_tools/types.tsx (shared with MCPServer/MCPTool), eliminating the playground→chat cross-module dependency. chat/types.ts and both playground components now import from mcp_tools/types. - Only persist accumulated MCP events when the stream completes cleanly; aborted or errored turns drop partial events to avoid showing incomplete tool calls. * fix(responses-api): use server_name for MCP URL routing, fix test path - Use server_name (not alias) as the URL path segment for MCP server_url; alias is a display name that may differ from the registered proxy route. URL-encode the path to handle names with spaces/special characters. - Fix sys.path.insert in tests to use __file__-relative path so tests pass regardless of which directory pytest is invoked from. * fix(chat-ui): fix stale session after failed edit, clean MCP event persistence, unique server_label - Eagerly call setResponsesSessionId(null) when historyOverride is set so a failed/aborted edit does not leave a stale session contaminating the next turn - Replace abort-signal check with streamCompletedCleanly flag to correctly skip MCP event persistence on both abort and non-abort errors (network/API failures) - Use server_name (unique) as server_label instead of alias to prevent silent tool-routing failures when two MCP servers share the same display name * [Feat] UI - Show logos on MCP Apps page (#23320) * feat(ui): add MCP server logo support across admin and chat UIs - New MCPLogoSelector component with grid of well-known logos (GitHub, Slack, Notion, Linear, Jira, etc.) and custom URL input - Create MCP Server form: logo picker with preview, OpenAPI presets auto-fill logo from registry icon_url - Edit MCP Server form: logo picker pre-populated from mcp_info.logo_url - Admin table: logos rendered next to server name in Name column - Chat MCPAppsPanel: logos on server cards (list + detail view) with graceful fallback to letter avatars - Chat MCPConnectPicker: logos next to server names in toggle list - Fix pre-existing bug: setTools -> clearTools in create form cancel - All 321 vitest files / 3211 tests pass Co-authored-by: Ishaan Jaff * feat(ui): use local SVG logos for MCP services, fix Chat UI rendering - Add 15 new MCP service logo SVGs (Slack, Notion, Linear, Jira, Figma, Gmail, Stripe, Salesforce, Shopify, HubSpot, Twilio, Sentry, Zapier, GitLab, Google Drive) to both source and pre-built directories - Switch MCPLogoSelector from CDN URLs (cdn.simpleicons.org) to local asset paths (/ui/assets/logos/) for reliable rendering - Logos now served by the proxy itself, working from any page path including /ui/chat/ (absolute paths resolve correctly everywhere) Co-authored-by: Ishaan Jaff --------- Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff * fix(codeql): remove ruby from language matrix (#23227) * Add team-scoped MCP server filtering for key creation and fix UnboundLocalError When creating a key, the MCP server list now filters by the selected team's allowed servers. Also fixes UnboundLocalError on `is_restricted_virtual_key` when `team_id` query param was provided to GET /v1/mcp/server. Co-Authored-By: Claude Opus 4.6 * Fix cross-team MCP server info disclosure and restricted key bypass The GET /v1/mcp/server endpoint allowed any authenticated user to pass an arbitrary team_id and enumerate another team's MCP server config. Restricted virtual keys could also use the team_id param to bypass their access limitations. Add team membership check for non-admins and block restricted keys from using the team_id filter. Co-Authored-By: Claude Opus 4.6 * Fix mcp_tool_permissions JSON string deserialization in _resolve_team_allowed_mcp_servers Co-Authored-By: Claude Opus 4.6 * [Feature] UI - MCP Servers: Add per-server health recheck Allow users to recheck health for individual MCP servers by clicking the health status badge. On hover the badge text changes to "Recheck" with a refresh icon, and the check runs only for that server. Co-Authored-By: Claude Opus 4.6 * Fix Anthropic docs link for beta endpoint Update the Anthropic /v1/messages beta endpoint docstring to point to its current pass-through documentation. This keeps the change scoped to the incorrect URL and avoids changing unverified wording in the surrounding comment. --------- Co-authored-by: netbrah <162479981+netbrah@users.noreply.github.com> Co-authored-by: Yong woo Song Co-authored-by: yuneng-jiang Co-authored-by: Claude Opus 4.6 Co-authored-by: Krish Dholakia Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff Co-authored-by: Sameer Kankute Co-authored-by: Ishaan Jaff Co-authored-by: Joe Reyna --- .github/workflows/codeql.yml | 2 - .../prompt_templates/factory.py | 48 +++ litellm/llms/anthropic/chat/transformation.py | 15 + ...odel_prices_and_context_window_backup.json | 86 +++++ .../_experimental/out/assets/logos/figma.svg | 7 + .../_experimental/out/assets/logos/gitlab.svg | 8 + .../_experimental/out/assets/logos/gmail.svg | 3 + .../out/assets/logos/google_drive.svg | 6 + .../out/assets/logos/hubspot.svg | 3 + .../_experimental/out/assets/logos/jira.svg | 15 + .../_experimental/out/assets/logos/linear.svg | 3 + .../_experimental/out/assets/logos/notion.svg | 3 + .../out/assets/logos/salesforce.svg | 3 + .../_experimental/out/assets/logos/sentry.svg | 3 + .../out/assets/logos/shopify.svg | 4 + .../_experimental/out/assets/logos/slack.svg | 6 + .../_experimental/out/assets/logos/stripe.svg | 3 + .../_experimental/out/assets/logos/twilio.svg | 3 + .../_experimental/out/assets/logos/zapier.svg | 3 + .../proxy/anthropic_endpoints/endpoints.py | 2 +- .../key_management_endpoints.py | 28 ++ .../mcp_management_endpoints.py | 126 ++++++- .../object_permission_utils.py | 184 ++++++++- model_prices_and_context_window.json | 86 +++++ ...llm_core_utils_prompt_templates_factory.py | 288 +++++++++++++- .../test_anthropic_chat_transformation.py | 125 +++++++ .../test_key_management_endpoints.py | 7 + .../test_mcp_management_endpoints.py | 151 ++++++++ .../test_object_permission_utils.py | 353 +++++++++++++++++- .../test_chat_ui_responses_session.py | 127 +++++++ .../public/assets/logos/figma.svg | 7 + .../public/assets/logos/gitlab.svg | 8 + .../public/assets/logos/gmail.svg | 3 + .../public/assets/logos/google_drive.svg | 6 + .../public/assets/logos/hubspot.svg | 3 + .../public/assets/logos/jira.svg | 15 + .../public/assets/logos/linear.svg | 3 + .../public/assets/logos/notion.svg | 3 + .../public/assets/logos/salesforce.svg | 3 + .../public/assets/logos/sentry.svg | 3 + .../public/assets/logos/shopify.svg | 4 + .../public/assets/logos/slack.svg | 6 + .../public/assets/logos/stripe.svg | 3 + .../public/assets/logos/twilio.svg | 3 + .../public/assets/logos/zapier.svg | 3 + .../app/(dashboard)/components/Sidebar2.tsx | 68 +++- .../hooks/mcpServers/useMCPServerHealth.ts | 41 +- .../hooks/mcpServers/useMCPServers.ts | 6 +- .../src/components/chat/ChatMessages.tsx | 10 + .../src/components/chat/ChatPage.tsx | 82 +++- .../src/components/chat/MCPAppsPanel.tsx | 37 +- .../src/components/chat/MCPConnectPicker.tsx | 12 + .../src/components/chat/types.ts | 4 + .../src/components/chat/useChatHistory.ts | 11 +- .../MCPServerSelector.tsx | 4 +- .../components/mcp_tools/MCPLogoSelector.tsx | 123 ++++++ .../mcp_tools/OpenAPIFormSection.tsx | 4 + .../mcp_tools/create_mcp_server.tsx | 12 + .../mcp_tools/mcp_server_columns.tsx | 166 +++++--- .../components/mcp_tools/mcp_server_edit.tsx | 4 + .../src/components/mcp_tools/mcp_servers.tsx | 6 +- .../src/components/mcp_tools/types.tsx | 27 ++ .../src/components/networking.tsx | 11 +- .../organisms/create_key_button.tsx | 3 + .../playground/chat_ui/MCPEventsDisplay.tsx | 27 +- .../playground/llm_calls/responses_api.tsx | 15 +- .../view_logs/ConfigInfoMessage.test.tsx | 41 ++ .../components/view_logs/ErrorViewer.test.tsx | 87 +++++ .../LogDetailsDrawer/TruncatedValue.test.tsx | 32 ++ .../components/view_logs/TypeBadges.test.tsx | 46 +++ .../components/view_logs/time_cell.test.tsx | 36 ++ 71 files changed, 2526 insertions(+), 163 deletions(-) create mode 100644 litellm/proxy/_experimental/out/assets/logos/figma.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/gitlab.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/gmail.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/google_drive.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/hubspot.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/jira.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/linear.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/notion.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/salesforce.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/sentry.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/shopify.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/slack.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/stripe.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/twilio.svg create mode 100644 litellm/proxy/_experimental/out/assets/logos/zapier.svg create mode 100644 tests/test_litellm/test_chat_ui_responses_session.py create mode 100644 ui/litellm-dashboard/public/assets/logos/figma.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/gitlab.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/gmail.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/google_drive.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/hubspot.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/jira.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/linear.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/notion.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/salesforce.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/sentry.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/shopify.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/slack.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/stripe.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/twilio.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/zapier.svg create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/MCPLogoSelector.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/ConfigInfoMessage.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/ErrorViewer.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/TruncatedValue.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3d11345e85..0b7cce2e4b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -34,8 +34,6 @@ jobs: build-mode: none - language: python build-mode: none - - language: ruby - build-mode: none steps: - name: Checkout repository diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index a694cec7d6..5e905da223 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2221,6 +2221,11 @@ def sanitize_messages_for_tool_calling( Case C: Empty text content - Replace empty or whitespace-only text content with a placeholder message. + Case D: Duplicate tool_result for same tool_use (duplicate results) + - If multiple tool messages reference the same tool_call_id, keep only the last + occurrence. Anthropic requires exactly one tool_result per tool_use and rejects + with: "each tool_use must have a single result". + This function operates on OpenAI format messages before they are converted to provider-specific formats. """ @@ -2256,6 +2261,49 @@ def sanitize_messages_for_tool_calling( sanitized_messages.append(current_message) i += 1 + # Case D: Deduplicate tool results with the same tool_call_id. + # Anthropic requires exactly one tool_result per tool_use. Session history + # (e.g. from conversation resume) can contain duplicate tool_result messages + # for the same tool_call_id. Keep only the last occurrence *within each + # contiguous block of tool results following an assistant message*. This + # avoids dropping results from earlier turns if a tool_call_id is reused. + # + # NOTE: This intentionally keeps the *last* occurrence (most complete for + # session-resume duplicates), unlike _deduplicate_bedrock_content_blocks + # which keeps the *first*. The Bedrock case handles provider-side content + # block duplication where the first is authoritative; here the duplicate + # arises from history replay where the last entry is the final state. + duplicates_to_remove: Set[int] = set() + seen_in_block: Dict[str, int] = {} # tool_call_id -> index (reset per block) + for idx, msg in enumerate(sanitized_messages): + role = msg.get("role") + tcid = msg.get("tool_call_id") if role in ["tool", "function"] else None + if tcid: + if tcid in seen_in_block: + # Mark the earlier occurrence for removal (keep latest) + duplicates_to_remove.add(seen_in_block[tcid]) + verbose_logger.warning( + "sanitize_messages_for_tool_calling: dropping duplicate " + "tool_result with tool_call_id=%s. This may indicate " + "duplicate tool messages in conversation history.", + tcid, + ) + seen_in_block[tcid] = idx + elif role not in ("tool", "function"): + # Non-tool message (user, assistant, system) marks a + # conversational-turn boundary — reset tracking. + # Tool/function messages with no tool_call_id are malformed; + # they should NOT reset the block because they don't represent + # a turn boundary and would mask real within-block duplicates. + seen_in_block = {} + + if duplicates_to_remove: + sanitized_messages = [ + msg + for idx, msg in enumerate(sanitized_messages) + if idx not in duplicates_to_remove + ] + return sanitized_messages diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 04b27e8782..fd1859f7d1 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -395,6 +395,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): }, ) + # Anthropic requires input_schema.type to be "object". Normalize + # schemas from external sources (MCP servers, OpenAI callers) that + # may omit the type field or use a non-object type. + if _input_schema.get("type") != "object": + litellm.verbose_logger.debug( + "_map_tool_helper: coercing input_schema type from %r to " + "'object' for Anthropic compatibility (tool: %s)", + _input_schema.get("type"), + tool["function"].get("name"), + ) + _input_schema = dict(_input_schema) # avoid mutating caller's dict + _input_schema["type"] = "object" + if "properties" not in _input_schema: + _input_schema["properties"] = {} + _allowed_properties = set(AnthropicInputSchema.__annotations__.keys()) input_schema_filtered = { k: v for k, v in _input_schema.items() if k in _allowed_properties diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bb4a678b54..d3dd6b3d99 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27665,6 +27665,92 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.5-35b-a3b": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-27b": { + "input_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-122b-a10b": { + "input_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-flash-02-23": { + "input_cost_per_token": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-plus-02-15": { + "input_cost_per_token": 4e-07, + "input_cost_per_token_above_256k_tokens": 5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_256k_tokens": 3e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-397b-a17b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/switchpoint/router": { "input_cost_per_token": 8.5e-07, "litellm_provider": "openrouter", diff --git a/litellm/proxy/_experimental/out/assets/logos/figma.svg b/litellm/proxy/_experimental/out/assets/logos/figma.svg new file mode 100644 index 0000000000..2d8b70457d --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/figma.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/gitlab.svg b/litellm/proxy/_experimental/out/assets/logos/gitlab.svg new file mode 100644 index 0000000000..18a89fa328 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/gitlab.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/gmail.svg b/litellm/proxy/_experimental/out/assets/logos/gmail.svg new file mode 100644 index 0000000000..d702890620 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/gmail.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/google_drive.svg b/litellm/proxy/_experimental/out/assets/logos/google_drive.svg new file mode 100644 index 0000000000..7048af9915 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/google_drive.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/hubspot.svg b/litellm/proxy/_experimental/out/assets/logos/hubspot.svg new file mode 100644 index 0000000000..b993945ac6 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/hubspot.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/jira.svg b/litellm/proxy/_experimental/out/assets/logos/jira.svg new file mode 100644 index 0000000000..fb10ca7517 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/jira.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/linear.svg b/litellm/proxy/_experimental/out/assets/logos/linear.svg new file mode 100644 index 0000000000..83662a1f9f --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/linear.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/notion.svg b/litellm/proxy/_experimental/out/assets/logos/notion.svg new file mode 100644 index 0000000000..170b9bb414 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/notion.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/salesforce.svg b/litellm/proxy/_experimental/out/assets/logos/salesforce.svg new file mode 100644 index 0000000000..1a541a004f --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/salesforce.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/sentry.svg b/litellm/proxy/_experimental/out/assets/logos/sentry.svg new file mode 100644 index 0000000000..9c3733dc43 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/sentry.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/shopify.svg b/litellm/proxy/_experimental/out/assets/logos/shopify.svg new file mode 100644 index 0000000000..fcc7547269 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/shopify.svg @@ -0,0 +1,4 @@ + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/slack.svg b/litellm/proxy/_experimental/out/assets/logos/slack.svg new file mode 100644 index 0000000000..801de4f70c --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/slack.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/stripe.svg b/litellm/proxy/_experimental/out/assets/logos/stripe.svg new file mode 100644 index 0000000000..ac16a6fb17 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/stripe.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/twilio.svg b/litellm/proxy/_experimental/out/assets/logos/twilio.svg new file mode 100644 index 0000000000..3517a2824d --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/twilio.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/_experimental/out/assets/logos/zapier.svg b/litellm/proxy/_experimental/out/assets/logos/zapier.svg new file mode 100644 index 0000000000..8428ba82a5 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/zapier.svg @@ -0,0 +1,3 @@ + + + diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 5b23b47923..c1f41467d6 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -30,7 +30,7 @@ async def anthropic_response( # noqa: PLR0915 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/anthropic_completion). + Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion). This was a BETA endpoint that calls 100+ LLMs in the anthropic format. """ diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 68f997e29c..b9dcc514d2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -64,6 +64,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, attach_object_permission_to_dict, handle_update_object_permission_common, + validate_key_mcp_servers_against_team, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -638,6 +639,12 @@ async def _common_key_generation_helper( # noqa: PLR0915 data_json.pop("tags") + # Validate MCP servers in object_permission are within team scope + await validate_key_mcp_servers_against_team( + object_permission=data_json.get("object_permission"), + team_obj=team_table, + ) + data_json = await _set_object_permission( data_json=data_json, prisma_client=prisma_client, @@ -1947,6 +1954,27 @@ async def update_key_fn( # Set Management Endpoint Metadata Fields + # Validate MCP servers in object_permission against the effective team + if data.object_permission is not None: + effective_team_obj = team_obj + # If team_id isn't being changed, resolve the existing key's team + if effective_team_obj is None and existing_key_row.team_id: + effective_team_obj = await get_team_object( + team_id=existing_key_row.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + check_db_only=True, + ) + object_permission_dict = ( + data.object_permission.model_dump() + if hasattr(data.object_permission, "model_dump") + else data.object_permission + ) + await validate_key_mcp_servers_against_team( + object_permission=object_permission_dict, + team_obj=effective_team_obj, + ) + non_default_values = await prepare_key_update_data( data=data, existing_key_row=existing_key_row ) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 46c59b4879..08f452859f 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -615,6 +615,46 @@ if MCP_AVAILABLE: return "view_all" return "restricted" + async def _get_team_scoped_mcp_server_list( + team_id: str, + ) -> List[LiteLLM_MCPServerTable]: + """ + Return MCP servers scoped to a team: team's allowed servers + allow_all_keys servers. + Used by the Create Key UI to populate the MCP server dropdown. + """ + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.management_helpers.object_permission_utils import ( + _get_allow_all_keys_server_ids, + _get_team_allowed_mcp_servers, + ) + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + team_obj = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + check_db_only=True, + ) + + team_server_ids = await _get_team_allowed_mcp_servers(team_obj) + allow_all_server_ids = _get_allow_all_keys_server_ids() + all_allowed_ids = team_server_ids | allow_all_server_ids + + if not all_allowed_ids: + return [] + + # Collect servers from registry + servers: List[LiteLLM_MCPServerTable] = [] + for server_id in all_allowed_ids: + server = global_mcp_server_manager.get_mcp_server_by_id(server_id) + if server is not None: + mcp_server_table = global_mcp_server_manager._build_mcp_server_table( + server + ) + servers.append(mcp_server_table) + + return _redact_mcp_credentials_list(servers) + @router.get( "/server", description="Returns the mcp server list with associated teams", @@ -623,38 +663,88 @@ if MCP_AVAILABLE: ) async def fetch_all_mcp_servers( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + team_id: Optional[str] = Query( + None, + description="Filter MCP servers by team scope. When provided, returns only " + "servers the team has access to plus globally available (allow_all_keys) servers. " + "Used by the Create Key UI to show team-scoped MCP servers.", + ), ): """ Get all of the configured mcp servers for the user in the db with their associated teams ``` curl --location 'http://localhost:4000/v1/mcp/server' \ --header 'Authorization: Bearer your_api_key_here' + + # Filter by team scope (for Create Key UI) + curl --location 'http://localhost:4000/v1/mcp/server?team_id=team-123' \ + --header 'Authorization: Bearer your_api_key_here' ``` """ - user_mcp_management_mode = _get_user_mcp_management_mode() + # If team_id is provided, return team-scoped servers + allow_all_keys servers is_restricted_virtual_key = _is_restricted_virtual_key_request( user_api_key_dict ) - - if user_mcp_management_mode == "view_all" and not is_restricted_virtual_key: - servers = await global_mcp_server_manager.get_all_mcp_servers_unfiltered() - redacted_mcp_servers = _redact_mcp_credentials_list(servers) - else: - auth_contexts = await build_effective_auth_contexts(user_api_key_dict) - - aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} - for auth_context in auth_contexts: - servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( - user_api_key_auth=auth_context + if team_id is not None and isinstance(team_id, str) and team_id.strip(): + # Restricted virtual keys must not use the team_id filter to + # bypass their own access limitations. + if is_restricted_virtual_key: + raise HTTPException( + status_code=403, + detail="Restricted virtual keys cannot query team-scoped MCP servers.", ) - for server in servers: - if server.server_id not in aggregated_servers: - aggregated_servers[server.server_id] = server - redacted_mcp_servers = _redact_mcp_credentials_list( - aggregated_servers.values() - ) + # Only proxy admins may query another team's MCP servers. + # Non-admins must belong to the requested team. + sanitized_team_id = team_id.strip() + is_admin = _user_has_admin_view(user_api_key_dict) + if not is_admin: + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.proxy_server import ( + prisma_client, + user_api_key_cache, + ) + + team_obj = await get_team_object( + team_id=sanitized_team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + check_db_only=True, + ) + user_in_team = any( + m.user_id is not None + and m.user_id == user_api_key_dict.user_id + for m in team_obj.members_with_roles + ) + if not user_in_team: + raise HTTPException( + status_code=403, + detail="You do not have permission to view MCP servers for this team.", + ) + + redacted_mcp_servers = await _get_team_scoped_mcp_server_list(sanitized_team_id) + else: + user_mcp_management_mode = _get_user_mcp_management_mode() + + if user_mcp_management_mode == "view_all" and not is_restricted_virtual_key: + servers = await global_mcp_server_manager.get_all_mcp_servers_unfiltered() + redacted_mcp_servers = _redact_mcp_credentials_list(servers) + else: + auth_contexts = await build_effective_auth_contexts(user_api_key_dict) + + aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {} + for auth_context in auth_contexts: + servers = await global_mcp_server_manager.get_all_allowed_mcp_servers( + user_api_key_auth=auth_context + ) + for server in servers: + if server.server_id not in aggregated_servers: + aggregated_servers[server.server_id] = server + + redacted_mcp_servers = _redact_mcp_credentials_list( + aggregated_servers.values() + ) # augment the mcp servers with public status if litellm.public_mcp_servers is not None: diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py index 9670cdf330..319a0b5eb7 100644 --- a/litellm/proxy/management_helpers/object_permission_utils.py +++ b/litellm/proxy/management_helpers/object_permission_utils.py @@ -4,12 +4,14 @@ organizations, teams, and keys. """ import json -from litellm._uuid import uuid -from typing import Dict, Optional, Union +from typing import Dict, List, Optional, Set, Union + +from fastapi import HTTPException, status from litellm._logging import verbose_proxy_logger -from litellm.proxy.utils import PrismaClient +from litellm._uuid import uuid from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.proxy.utils import PrismaClient @@ -177,4 +179,178 @@ async def _set_object_permission( data_json["object_permission_id"] = created_permission.object_permission_id data_json.pop("object_permission") - return data_json \ No newline at end of file + return data_json + + +async def _resolve_team_allowed_mcp_servers( + team_object_permission: "LiteLLM_ObjectPermissionTable", +) -> Set[str]: + """ + Resolve the full set of MCP server IDs a team has access to. + + Combines: + - Direct mcp_servers list + - Servers from mcp_access_groups + - Server IDs referenced in mcp_tool_permissions keys + """ + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + + direct_servers: List[str] = team_object_permission.mcp_servers or [] + access_group_servers: List[str] = ( + await MCPRequestHandler._get_mcp_servers_from_access_groups( + team_object_permission.mcp_access_groups or [] + ) + ) + raw_tool_perms = team_object_permission.mcp_tool_permissions or {} + if isinstance(raw_tool_perms, str): + raw_tool_perms = json.loads(raw_tool_perms) + tool_perm_servers: List[str] = list(raw_tool_perms.keys()) + return set(direct_servers + access_group_servers + tool_perm_servers) + + +def _get_allow_all_keys_server_ids() -> Set[str]: + """Return the set of MCP server IDs marked with allow_all_keys=True.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + return set(global_mcp_server_manager.get_allow_all_keys_server_ids()) + + +async def _get_team_allowed_mcp_servers( + team_obj: Optional["LiteLLM_TeamTableCachedObj"], +) -> Set[str]: + """ + Get the full set of MCP server IDs a team allows. + + If team has no object_permission or no MCP config, returns empty set + (meaning only allow_all_keys servers are permitted). + """ + if team_obj is None: + return set() + + team_object_permission = team_obj.object_permission + if team_object_permission is None: + return set() + + return await _resolve_team_allowed_mcp_servers(team_object_permission) + + +def _extract_requested_mcp_server_ids( + object_permission: Optional[dict], +) -> Set[str]: + """ + Extract all MCP server IDs referenced in a key's object_permission dict. + + Includes: + - mcp_servers list + - Keys from mcp_tool_permissions + """ + if not object_permission or not isinstance(object_permission, dict): + return set() + + server_ids: Set[str] = set() + mcp_servers = object_permission.get("mcp_servers") + if isinstance(mcp_servers, list): + server_ids.update(mcp_servers) + + mcp_tool_permissions = object_permission.get("mcp_tool_permissions") + if isinstance(mcp_tool_permissions, dict): + server_ids.update(mcp_tool_permissions.keys()) + + return server_ids + + +def _extract_requested_mcp_access_groups( + object_permission: Optional[dict], +) -> Set[str]: + """Extract MCP access groups from a key's object_permission dict.""" + if not object_permission or not isinstance(object_permission, dict): + return set() + + groups = object_permission.get("mcp_access_groups") + if isinstance(groups, list): + return set(groups) + return set() + + +async def validate_key_mcp_servers_against_team( + object_permission: Optional[dict], + team_obj: Optional["LiteLLM_TeamTableCachedObj"], +): + """ + Validate that MCP servers requested on a key are within the allowed scope. + + Rules: + - If key is in a team: key's mcp_servers must be a subset of + (team's allowed servers + allow_all_keys servers) + - If key is NOT in a team: key's mcp_servers must only contain + allow_all_keys servers + - If team has no MCP config: key can only use allow_all_keys servers + + Raises HTTPException(403) if validation fails. + """ + requested_servers = _extract_requested_mcp_server_ids(object_permission) + requested_access_groups = _extract_requested_mcp_access_groups(object_permission) + + # Nothing to validate + if not requested_servers and not requested_access_groups: + return + + allow_all_keys_servers = _get_allow_all_keys_server_ids() + team_allowed_servers = await _get_team_allowed_mcp_servers(team_obj) + + # Combined allowed set = team servers + allow_all_keys servers + all_allowed_servers = team_allowed_servers | allow_all_keys_servers + + # Validate requested server IDs + if requested_servers: + disallowed_servers = requested_servers - all_allowed_servers + if disallowed_servers: + if team_obj is not None: + detail = ( + f"Key requests MCP servers not allowed by team '{team_obj.team_id}': " + f"{sorted(disallowed_servers)}. " + f"Team allows: {sorted(team_allowed_servers)}. " + f"Global (allow_all_keys) servers: {sorted(allow_all_keys_servers)}." + ) + else: + detail = ( + f"Key is not in a team. Only globally available (allow_all_keys) MCP servers " + f"can be assigned: {sorted(allow_all_keys_servers)}. " + f"Disallowed servers: {sorted(disallowed_servers)}." + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": detail}, + ) + + # Validate requested access groups (must be subset of team's access groups) + if requested_access_groups: + team_access_groups: Set[str] = set() + if ( + team_obj is not None + and team_obj.object_permission is not None + and team_obj.object_permission.mcp_access_groups + ): + team_access_groups = set(team_obj.object_permission.mcp_access_groups) + + disallowed_groups = requested_access_groups - team_access_groups + if disallowed_groups: + if team_obj is not None: + detail = ( + f"Key requests MCP access groups not allowed by team '{team_obj.team_id}': " + f"{sorted(disallowed_groups)}. " + f"Team allows: {sorted(team_access_groups)}." + ) + else: + detail = ( + f"Key is not in a team. MCP access groups cannot be assigned to " + f"keys outside of a team. Disallowed groups: {sorted(disallowed_groups)}." + ) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"error": detail}, + ) \ No newline at end of file diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bb4a678b54..d3dd6b3d99 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27665,6 +27665,92 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/qwen/qwen3.5-35b-a3b": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-35b-a3b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-27b": { + "input_cost_per_token": 3e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-122b-a10b": { + "input_cost_per_token": 4e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-122b-a10b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-flash-02-23": { + "input_cost_per_token": 1e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://openrouter.ai/qwen/qwen3.5-flash-02-23", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-plus-02-15": { + "input_cost_per_token": 4e-07, + "input_cost_per_token_above_256k_tokens": 5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.4e-06, + "output_cost_per_token_above_256k_tokens": 3e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-plus-02-15", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "openrouter/qwen/qwen3.5-397b-a17b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "source": "https://openrouter.ai/qwen/qwen3.5-397b-a17b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "openrouter/switchpoint/router": { "input_cost_per_token": 8.5e-07, "litellm_provider": "openrouter", diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 707b5bdc77..8d68539564 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockImageProcessor, _convert_to_bedrock_tool_call_invoke, ollama_pt, + sanitize_messages_for_tool_calling, ) @@ -1179,7 +1180,7 @@ def test_bedrock_tools_pt_does_not_handle_system_tool(): System tools (nova_grounding) should be added via web_search_options, not via the tools parameter directly. """ - + from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt # Regular function tools should still work @@ -1741,3 +1742,288 @@ def test_bedrock_tool_call_invoke_multiple_normal_tools(): assert len(result) == 2 assert result[0]["toolUse"]["toolUseId"] == "call_1" assert result[1]["toolUse"]["toolUseId"] == "call_2" + + +# ======================================================================== +# Tool result deduplication tests (Case D in sanitize_messages_for_tool_calling) +# ======================================================================== + + +def test_sanitize_messages_deduplicates_tool_results(): + """ + Anthropic requires exactly one tool_result per tool_use. When conversation + history (e.g. from session resume) contains duplicate tool result messages + with the same tool_call_id, sanitize_messages_for_tool_calling should keep + only the last occurrence. + + Without this fix, Anthropic rejects with: + each tool_use must have a single result. Found multiple tool_result + blocks with id: + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + } + ], + }, + # First tool result (stale/duplicate) + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "Partial result...", + }, + # Second tool result (final/complete — should be kept) + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": '{"temperature": 72, "condition": "sunny"}', + }, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Count tool messages with this ID — should be exactly 1 + tool_results = [ + m for m in result if m.get("role") == "tool" and m.get("tool_call_id") == "call_abc123" + ] + assert len(tool_results) == 1 + # Should keep the LAST occurrence (most complete) + assert tool_results[0]["content"] == '{"temperature": 72, "condition": "sunny"}' + finally: + litellm.modify_params = original + + +def test_sanitize_messages_preserves_unique_tool_results(): + """ + When each tool_call_id has exactly one tool_result, no deduplication should + occur. Messages should pass through unchanged. + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "Get weather for two cities"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "LA"}', + }, + }, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "72F"}, + {"role": "tool", "tool_call_id": "call_2", "content": "85F"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + tool_results = [m for m in result if m.get("role") == "tool"] + assert len(tool_results) == 2 + assert tool_results[0]["tool_call_id"] == "call_1" + assert tool_results[0]["content"] == "72F" + assert tool_results[1]["tool_call_id"] == "call_2" + assert tool_results[1]["content"] == "85F" + finally: + litellm.modify_params = original + + +def test_sanitize_messages_dedup_disabled_when_modify_params_false(): + """ + When litellm.modify_params is False, messages should be returned as-is + even if they contain duplicate tool results. + """ + original = litellm.modify_params + litellm.modify_params = False + try: + messages = [ + {"role": "user", "content": "Test"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_dup", + "type": "function", + "function": {"name": "test", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_dup", "content": "first"}, + {"role": "tool", "tool_call_id": "call_dup", "content": "second"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Should be unchanged — no sanitization when modify_params=False + assert result == messages + finally: + litellm.modify_params = original + + +def test_sanitize_messages_dedup_scoped_per_turn_preserves_cross_turn(): + """ + When the same tool_call_id appears in two different assistant turns + (separated by a user message), both tool results must be preserved. + Deduplication should only apply within a single contiguous tool-result + block, not globally across the conversation. + + Without per-turn scoping this would incorrectly drop the first tool result, + leaving the first assistant message without its required result (which + Anthropic would reject). + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "First question"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_X", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q": "a"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_X", "content": "result_turn_1"}, + {"role": "user", "content": "Second question"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_X", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q": "b"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_X", "content": "result_turn_2"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Both tool results must survive — one per turn + tool_results = [ + m for m in result + if m.get("role") == "tool" and m.get("tool_call_id") == "call_X" + ] + assert len(tool_results) == 2, ( + f"Expected 2 tool results (one per turn), got {len(tool_results)}. " + "Dedup may be global instead of per-turn scoped." + ) + assert tool_results[0]["content"] == "result_turn_1" + assert tool_results[1]["content"] == "result_turn_2" + finally: + litellm.modify_params = original + + +def test_sanitize_messages_combined_case_a_and_case_d(): + """ + Combined Case A + Case D: an assistant message has two tool_calls — + one with a missing result (Case A should inject a dummy) and one with + duplicate results (Case D should deduplicate to keep only the last). + + This validates that both sanitization passes compose correctly without + interfering with each other. + """ + original = litellm.modify_params + litellm.modify_params = True + try: + messages = [ + {"role": "user", "content": "Do two things"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_missing", + "type": "function", + "function": {"name": "tool_a", "arguments": "{}"}, + }, + { + "id": "call_duped", + "type": "function", + "function": {"name": "tool_b", "arguments": '{"q": "x"}'}, + }, + ], + }, + # No result for call_missing — Case A should inject a dummy + # Duplicate results for call_duped — Case D should keep last + {"role": "tool", "tool_call_id": "call_duped", "content": "stale_result"}, + {"role": "tool", "tool_call_id": "call_duped", "content": "fresh_result"}, + {"role": "user", "content": "Now summarize"}, + ] + + result = sanitize_messages_for_tool_calling(messages) + + # Collect tool results from the output + tool_results = [m for m in result if m.get("role") in ("tool", "function")] + + # Case A: call_missing should have a dummy result injected + missing_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_missing" + ] + assert len(missing_results) == 1, ( + f"Expected 1 dummy result for call_missing (Case A), got {len(missing_results)}" + ) + + # Case D: call_duped should have exactly 1 result (the fresh one) + duped_results = [ + m for m in tool_results if m.get("tool_call_id") == "call_duped" + ] + assert len(duped_results) == 1, ( + f"Expected 1 result for call_duped after dedup (Case D), got {len(duped_results)}" + ) + assert duped_results[0]["content"] == "fresh_result", ( + f"Expected last-wins 'fresh_result', got '{duped_results[0]['content']}'" + ) + + # Verify tool results immediately follow the assistant message + asst_idx = next( + i for i, m in enumerate(result) if m.get("role") == "assistant" + ) + tool_msgs_after_asst = [ + m + for m in result[asst_idx + 1 :] + if m.get("role") in ("tool", "function") + ] + assert len(tool_msgs_after_asst) == 2, ( + f"Expected 2 tool results after assistant, got {len(tool_msgs_after_asst)}" + ) + # Both tool_call_ids should be present (order may vary) + tool_ids = {m["tool_call_id"] for m in tool_msgs_after_asst} + assert tool_ids == {"call_missing", "call_duped"}, ( + f"Expected tool_call_ids {{call_missing, call_duped}}, got {tool_ids}" + ) + finally: + litellm.modify_params = original diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index b540b0d952..6f03f630b5 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3175,3 +3175,128 @@ def test_map_openai_params_max_tokens_normalized_to_int(): assert "max_tokens" in result assert result["max_tokens"] == 1 + + +# ======================================================================== +# Tool schema normalization tests +# ======================================================================== + + +def test_map_tool_helper_enforces_object_type_when_missing(): + """ + Anthropic requires input_schema.type to be "object". When an OpenAI tool + has parameters without a 'type' field (common with MCP servers), LiteLLM + should inject type:"object" before forwarding to Anthropic. + + Without this fix, Anthropic rejects with: + tools.N.custom.input_schema.type: Input should be 'object' + """ + config = AnthropicConfig() + + # Tool with parameters that has properties but no 'type' field + tool = { + "type": "function", + "function": { + "name": "search_code", + "description": "Search for code patterns", + "parameters": { + "properties": { + "query": {"type": "string", "description": "Search query"} + }, + "required": ["query"], + }, + }, + } + + original_params = tool["function"]["parameters"].copy() + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert "properties" in result["input_schema"] + assert "query" in result["input_schema"]["properties"] + # Original parameters dict must not be modified in place + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) + + +def test_map_tool_helper_enforces_object_type_when_wrong_type(): + """ + If a tool schema has type:"string" or type:"array" at the root level, + LiteLLM should normalize it to type:"object" for Anthropic compatibility. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "echo", + "description": "Echo input", + "parameters": { + "type": "string", + "description": "The input to echo", + }, + }, + } + + original_params = tool["function"]["parameters"].copy() + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert result["input_schema"].get("properties") == {}, ( + "properties should be injected as {} when schema has non-object type and no properties key" + ) + # Original parameters dict must not be modified in place + assert tool["function"]["parameters"] == original_params, ( + "parameters dict was mutated; _map_tool_helper should not modify caller data" + ) + + +def test_map_tool_helper_preserves_valid_object_schema(): + """ + When a tool schema already has type:"object", it should be preserved + without modification. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + }, + "required": ["city"], + }, + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert "city" in result["input_schema"]["properties"] + assert result["input_schema"]["required"] == ["city"] + + +def test_map_tool_helper_empty_parameters_get_default(): + """ + When parameters is entirely missing, the existing default should still + produce a valid {type:"object", properties:{}} schema. + """ + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "no_params_tool", + "description": "Tool with no parameters", + }, + } + + result, _ = config._map_tool_helper(tool) + assert result is not None + assert result["input_schema"]["type"] == "object" + assert result["input_schema"].get("properties") == {} diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 55366bbec2..09dfdb81cb 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -605,6 +605,10 @@ async def test_key_generation_with_mcp_tool_permissions(monkeypatch): mock_prisma_client.insert_data = AsyncMock(side_effect=_insert_data_side_effect) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_mcp_servers_against_team", + AsyncMock(), + ) from litellm.proxy._types import ( GenerateKeyRequest, @@ -2346,6 +2350,9 @@ async def test_generate_key_with_object_permission(): ), patch( "litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin", + ), patch( + "litellm.proxy.management_endpoints.key_management_endpoints.validate_key_mcp_servers_against_team", + new_callable=AsyncMock, ): # Execute result = await _common_key_generation_helper( diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 30b3be4a3e..ea51965ebf 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -797,6 +797,157 @@ class TestListMCPServers: assert result.status == "healthy" +class TestTeamScopedMCPServerAccess: + """Tests for cross-team information disclosure and restricted key bypass fixes.""" + + @pytest.mark.asyncio + async def test_non_member_cannot_query_foreign_team(self): + """Non-admin user who is NOT a member of the target team should get 403.""" + from litellm.proxy._types import Member + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="attacker_user", + ) + + # Team with a different member + mock_team_obj = MagicMock() + mock_team_obj.members_with_roles = [ + Member(user_id="legitimate_user", role="admin"), + ] + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=False, + ), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + AsyncMock(return_value=mock_team_obj), + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + with pytest.raises(HTTPException) as exc_info: + await fetch_all_mcp_servers( + user_api_key_dict=mock_user_auth, team_id="foreign-team-id" + ) + assert exc_info.value.status_code == 403 + assert "permission" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_team_member_can_query_own_team(self): + """User who IS a member of the team should be able to query it.""" + from litellm.proxy._types import Member + + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="team_member", + ) + + mock_team_obj = MagicMock() + mock_team_obj.members_with_roles = [ + Member(user_id="team_member", role="user"), + ] + mock_team_obj.object_permission = MagicMock(mcp_servers=["server-1"]) + + mock_server = generate_mock_mcp_server_config_record( + server_id="server-1", name="Team Server" + ) + mock_manager = MagicMock() + mock_manager.get_mcp_server_by_id = MagicMock(return_value=mock_server) + mock_manager._build_mcp_server_table = MagicMock( + return_value=generate_mock_mcp_server_db_record(server_id="server-1") + ) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=False, + ), + patch( + "litellm.proxy.auth.auth_checks.get_team_object", + AsyncMock(return_value=mock_team_obj), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list", + AsyncMock( + return_value=[ + generate_mock_mcp_server_db_record(server_id="server-1") + ] + ), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager", + mock_manager, + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + result = await fetch_all_mcp_servers( + user_api_key_dict=mock_user_auth, team_id="my-team-id" + ) + assert len(result) == 1 + assert result[0].server_id == "server-1" + + @pytest.mark.asyncio + async def test_admin_can_query_any_team(self): + """Proxy admins should be able to query any team's MCP servers.""" + mock_user_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="admin_user", + ) + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_has_admin_view", + return_value=True, + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_team_scoped_mcp_server_list", + AsyncMock( + return_value=[ + generate_mock_mcp_server_db_record(server_id="server-1") + ] + ), + ), + ): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + # Admin should NOT need to be a team member + result = await fetch_all_mcp_servers( + user_api_key_dict=mock_user_auth, team_id="any-team-id" + ) + assert len(result) == 1 + + @pytest.mark.asyncio + async def test_restricted_virtual_key_cannot_use_team_id_filter(self): + """Restricted virtual keys must not bypass access limits via team_id.""" + mock_user_auth = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="vkey_user", + api_key="sk-restricted", + allowed_routes=["mcp_routes"], + ) + + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + fetch_all_mcp_servers, + ) + + with pytest.raises(HTTPException) as exc_info: + await fetch_all_mcp_servers( + user_api_key_dict=mock_user_auth, team_id="some-team" + ) + assert exc_info.value.status_code == 403 + assert "Restricted virtual key" in str(exc_info.value.detail) + + class TestTemporaryMCPSessionEndpoints: def test_inherit_credentials_from_existing_server(self): payload = NewMCPServerRequest( diff --git a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py index 07d89035dc..202b95b319 100644 --- a/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py +++ b/tests/test_litellm/proxy/management_helpers/test_object_permission_utils.py @@ -3,15 +3,21 @@ import os import sys import pytest +from fastapi import HTTPException sys.path.insert( 0, os.path.abspath("../../../..") ) -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch +from litellm.proxy._types import LiteLLM_ObjectPermissionTable from litellm.proxy.management_helpers.object_permission_utils import ( + _extract_requested_mcp_access_groups, + _extract_requested_mcp_server_ids, + _resolve_team_allowed_mcp_servers, _set_object_permission, + validate_key_mcp_servers_against_team, ) @@ -82,3 +88,348 @@ async def test_set_object_permission(): assert result["user_id"] == "test_user" assert result["models"] == ["gpt-4"] + +# ---- Tests for _extract_requested_mcp_server_ids ---- + + +def test_extract_requested_mcp_server_ids_from_mcp_servers(): + obj_perm = {"mcp_servers": ["server-1", "server-2"]} + assert _extract_requested_mcp_server_ids(obj_perm) == {"server-1", "server-2"} + + +def test_extract_requested_mcp_server_ids_from_tool_permissions(): + obj_perm = {"mcp_tool_permissions": {"server-a": ["tool1"], "server-b": ["tool2"]}} + assert _extract_requested_mcp_server_ids(obj_perm) == {"server-a", "server-b"} + + +def test_extract_requested_mcp_server_ids_combined(): + obj_perm = { + "mcp_servers": ["server-1"], + "mcp_tool_permissions": {"server-2": ["tool1"]}, + } + assert _extract_requested_mcp_server_ids(obj_perm) == {"server-1", "server-2"} + + +def test_extract_requested_mcp_server_ids_none(): + assert _extract_requested_mcp_server_ids(None) == set() + assert _extract_requested_mcp_server_ids({}) == set() + + +# ---- Tests for _extract_requested_mcp_access_groups ---- + + +def test_extract_requested_mcp_access_groups(): + obj_perm = {"mcp_access_groups": ["group-a", "group-b"]} + assert _extract_requested_mcp_access_groups(obj_perm) == {"group-a", "group-b"} + + +def test_extract_requested_mcp_access_groups_none(): + assert _extract_requested_mcp_access_groups(None) == set() + assert _extract_requested_mcp_access_groups({}) == set() + + +# ---- Tests for validate_key_mcp_servers_against_team ---- + + +def _make_team_obj( + team_id="team-1", + mcp_servers=None, + mcp_access_groups=None, + mcp_tool_permissions=None, +): + """Create a mock team object with the given MCP permissions.""" + mock_team = MagicMock() + mock_team.team_id = team_id + + if mcp_servers is not None or mcp_access_groups is not None or mcp_tool_permissions is not None: + mock_team.object_permission = MagicMock(spec=LiteLLM_ObjectPermissionTable) + mock_team.object_permission.mcp_servers = mcp_servers or [] + mock_team.object_permission.mcp_access_groups = mcp_access_groups or [] + mock_team.object_permission.mcp_tool_permissions = mcp_tool_permissions or {} + else: + mock_team.object_permission = None + + return mock_team + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_no_object_permission(mock_access_groups, mock_allow_all): + """No object_permission on key — should pass without error.""" + await validate_key_mcp_servers_against_team( + object_permission=None, + team_obj=_make_team_obj(mcp_servers=["server-1"]), + ) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_key_servers_within_team_scope(mock_access_groups, mock_allow_all): + """Key requests servers that are in the team's scope — should pass.""" + team_obj = _make_team_obj(mcp_servers=["server-1", "server-2", "server-3"]) + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-1", "server-2"]}, + team_obj=team_obj, + ) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_key_servers_outside_team_scope_raises(mock_access_groups, mock_allow_all): + """Key requests servers NOT in the team's scope — should raise 403.""" + team_obj = _make_team_obj(mcp_servers=["server-1"]) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-1", "server-outside"]}, + team_obj=team_obj, + ) + assert exc_info.value.status_code == 403 + assert "server-outside" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value={"global-server"}, +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_allow_all_keys_servers_always_allowed(mock_access_groups, mock_allow_all): + """allow_all_keys servers should be accessible even if not in team scope.""" + team_obj = _make_team_obj(mcp_servers=["server-1"]) + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-1", "global-server"]}, + team_obj=team_obj, + ) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value={"global-server"}, +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_no_team_only_allow_all_keys(mock_access_groups, mock_allow_all): + """Key without a team can only use allow_all_keys servers.""" + # This should pass — requesting a global server without a team + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["global-server"]}, + team_obj=None, + ) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value={"global-server"}, +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_no_team_non_global_server_raises(mock_access_groups, mock_allow_all): + """Key without a team requesting a non-global server — should raise 403.""" + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["private-server"]}, + team_obj=None, + ) + assert exc_info.value.status_code == 403 + assert "not in a team" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_team_no_mcp_config_blocks_all(mock_access_groups, mock_allow_all): + """Team with no object_permission — key can't use any non-global MCP servers.""" + team_obj = _make_team_obj() # No object_permission + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["some-server"]}, + team_obj=team_obj, + ) + assert exc_info.value.status_code == 403 + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_tool_permissions_validated_against_team(mock_access_groups, mock_allow_all): + """Server IDs in mcp_tool_permissions should also be validated.""" + team_obj = _make_team_obj(mcp_servers=["server-1"]) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={ + "mcp_tool_permissions": {"server-outside": ["tool1"]} + }, + team_obj=team_obj, + ) + assert exc_info.value.status_code == 403 + assert "server-outside" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_access_groups_within_team_scope(mock_access_groups, mock_allow_all): + """Key requests access groups that are in the team's scope — should pass.""" + team_obj = _make_team_obj(mcp_access_groups=["group-a", "group-b"]) + await validate_key_mcp_servers_against_team( + object_permission={"mcp_access_groups": ["group-a"]}, + team_obj=team_obj, + ) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_access_groups_outside_team_scope_raises(mock_access_groups, mock_allow_all): + """Key requests access groups NOT in the team's scope — should raise 403.""" + team_obj = _make_team_obj(mcp_access_groups=["group-a"]) + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_access_groups": ["group-outside"]}, + team_obj=team_obj, + ) + assert exc_info.value.status_code == 403 + assert "group-outside" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_validate_access_groups_no_team_raises(mock_access_groups, mock_allow_all): + """Key without a team requesting access groups — should raise 403.""" + with pytest.raises(HTTPException) as exc_info: + await validate_key_mcp_servers_against_team( + object_permission={"mcp_access_groups": ["group-a"]}, + team_obj=None, + ) + assert exc_info.value.status_code == 403 + assert "not in a team" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +@patch( + "litellm.proxy.management_helpers.object_permission_utils._get_allow_all_keys_server_ids", + return_value=set(), +) +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=["server-from-group"], +) +async def test_validate_team_access_groups_resolve_to_servers(mock_access_groups, mock_allow_all): + """Team access groups should resolve to server IDs and be included in allowed set.""" + team_obj = _make_team_obj(mcp_access_groups=["group-a"]) + # Key requests a server that comes from the team's access group + await validate_key_mcp_servers_against_team( + object_permission={"mcp_servers": ["server-from-group"]}, + team_obj=team_obj, + ) + + +# ---- Tests for _resolve_team_allowed_mcp_servers with JSON string mcp_tool_permissions ---- + + +@pytest.mark.asyncio +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_resolve_team_allowed_mcp_servers_string_tool_permissions(mock_access_groups): + """mcp_tool_permissions stored as a JSON string (via safe_dumps) should be deserialized correctly.""" + mock_perm = MagicMock(spec=LiteLLM_ObjectPermissionTable) + mock_perm.mcp_servers = ["server-1"] + mock_perm.mcp_access_groups = [] + mock_perm.mcp_tool_permissions = json.dumps({"server-2": ["tool1"]}) + + result = await _resolve_team_allowed_mcp_servers(mock_perm) + assert result == {"server-1", "server-2"} + + +@pytest.mark.asyncio +@patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.MCPRequestHandler._get_mcp_servers_from_access_groups", + new_callable=AsyncMock, + return_value=[], +) +async def test_resolve_team_allowed_mcp_servers_dict_tool_permissions(mock_access_groups): + """mcp_tool_permissions as a dict should work without deserialization.""" + mock_perm = MagicMock(spec=LiteLLM_ObjectPermissionTable) + mock_perm.mcp_servers = [] + mock_perm.mcp_access_groups = [] + mock_perm.mcp_tool_permissions = {"server-a": ["tool1"]} + + result = await _resolve_team_allowed_mcp_servers(mock_perm) + assert result == {"server-a"} + diff --git a/tests/test_litellm/test_chat_ui_responses_session.py b/tests/test_litellm/test_chat_ui_responses_session.py new file mode 100644 index 0000000000..09ef003ebd --- /dev/null +++ b/tests/test_litellm/test_chat_ui_responses_session.py @@ -0,0 +1,127 @@ +""" +Tests for responses API session chaining used by the chat UI. + +Verifies that: +1. previous_response_id is correctly forwarded when provided +2. Absence of previous_response_id does not break the call +3. The aresponses function signature exposes the expected parameters +""" +import inspect +import json +import os +import sys +import unittest.mock as mock + +# Use __file__ so the import path is correct regardless of the pytest working directory. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +import httpx +import pytest + +import litellm + + +class TestResponsesSessionChaining: + """Test previous_response_id session chaining for the chat UI.""" + + def test_responses_api_signature_accepts_previous_response_id(self): + """aresponses must accept previous_response_id and onResponseId-like params.""" + sig = inspect.signature(litellm.aresponses) + assert "previous_response_id" in sig.parameters, ( + "aresponses must accept previous_response_id for multi-turn session chaining" + ) + assert "input" in sig.parameters, "aresponses must accept input" + assert "model" in sig.parameters, "aresponses must accept model" + + @pytest.mark.asyncio + async def test_previous_response_id_included_in_request_body(self): + """previous_response_id must appear in the outgoing HTTP request body.""" + captured_body: dict = {} + + async def mock_send(self_transport, request: httpx.Request, **kwargs): + try: + captured_body.update(json.loads(request.content)) + except Exception: + pass + # Return a minimal valid responses API response + response_json = { + "id": "resp_test123", + "object": "response", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_001", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + "status": "completed", + } + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, + "status": "completed", + "created_at": 1700000000, + } + return httpx.Response( + 200, + json=response_json, + request=request, + ) + + with mock.patch("httpx.AsyncClient.send", mock_send): + try: + await litellm.aresponses( + input="hello", + model="gpt-4o-mini", + previous_response_id="resp_prev_abc", + api_key="sk-test-fake", + ) + except Exception: + pass # response parsing may fail; we only care about the outgoing body + + assert captured_body.get("previous_response_id") == "resp_prev_abc", ( + f"Expected previous_response_id in request body, got: {captured_body}" + ) + + @pytest.mark.asyncio + async def test_no_previous_response_id_omitted_from_request(self): + """When previous_response_id is None, it must not appear in the request body.""" + captured_body: dict = {} + + async def mock_send(self_transport, request: httpx.Request, **kwargs): + try: + captured_body.update(json.loads(request.content)) + except Exception: + pass + response_json = { + "id": "resp_new001", + "object": "response", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_001", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi", "annotations": []}], + "status": "completed", + } + ], + "usage": {"input_tokens": 5, "output_tokens": 3, "total_tokens": 8}, + "status": "completed", + "created_at": 1700000000, + } + return httpx.Response(200, json=response_json, request=request) + + with mock.patch("httpx.AsyncClient.send", mock_send): + try: + await litellm.aresponses( + input="hello", + model="gpt-4o-mini", + previous_response_id=None, + api_key="sk-test-fake", + ) + except Exception: + pass + + assert "previous_response_id" not in captured_body, ( + "previous_response_id must be omitted from the request body when None" + ) diff --git a/ui/litellm-dashboard/public/assets/logos/figma.svg b/ui/litellm-dashboard/public/assets/logos/figma.svg new file mode 100644 index 0000000000..2d8b70457d --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/figma.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/gitlab.svg b/ui/litellm-dashboard/public/assets/logos/gitlab.svg new file mode 100644 index 0000000000..18a89fa328 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/gitlab.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/gmail.svg b/ui/litellm-dashboard/public/assets/logos/gmail.svg new file mode 100644 index 0000000000..d702890620 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/gmail.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/google_drive.svg b/ui/litellm-dashboard/public/assets/logos/google_drive.svg new file mode 100644 index 0000000000..7048af9915 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/google_drive.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/hubspot.svg b/ui/litellm-dashboard/public/assets/logos/hubspot.svg new file mode 100644 index 0000000000..b993945ac6 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/hubspot.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/jira.svg b/ui/litellm-dashboard/public/assets/logos/jira.svg new file mode 100644 index 0000000000..fb10ca7517 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/jira.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/linear.svg b/ui/litellm-dashboard/public/assets/logos/linear.svg new file mode 100644 index 0000000000..83662a1f9f --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/linear.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/notion.svg b/ui/litellm-dashboard/public/assets/logos/notion.svg new file mode 100644 index 0000000000..170b9bb414 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/notion.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/salesforce.svg b/ui/litellm-dashboard/public/assets/logos/salesforce.svg new file mode 100644 index 0000000000..1a541a004f --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/salesforce.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/sentry.svg b/ui/litellm-dashboard/public/assets/logos/sentry.svg new file mode 100644 index 0000000000..9c3733dc43 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/sentry.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/shopify.svg b/ui/litellm-dashboard/public/assets/logos/shopify.svg new file mode 100644 index 0000000000..fcc7547269 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/shopify.svg @@ -0,0 +1,4 @@ + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/slack.svg b/ui/litellm-dashboard/public/assets/logos/slack.svg new file mode 100644 index 0000000000..801de4f70c --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/slack.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/ui/litellm-dashboard/public/assets/logos/stripe.svg b/ui/litellm-dashboard/public/assets/logos/stripe.svg new file mode 100644 index 0000000000..ac16a6fb17 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/stripe.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/twilio.svg b/ui/litellm-dashboard/public/assets/logos/twilio.svg new file mode 100644 index 0000000000..3517a2824d --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/twilio.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/public/assets/logos/zapier.svg b/ui/litellm-dashboard/public/assets/logos/zapier.svg new file mode 100644 index 0000000000..8428ba82a5 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/zapier.svg @@ -0,0 +1,3 @@ + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx index dbc1c4d10e..ceb14864ad 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx @@ -20,6 +20,7 @@ import { ToolOutlined, TagsOutlined, AuditOutlined, + MessageOutlined, } from "@ant-design/icons"; // import { // all_admin_roles, @@ -47,6 +48,7 @@ interface SidebarProps { interface MenuItemCfg { key: string; + newTab?: boolean; page: string; // legacy id; we map this to a path below label: string; roles?: string[]; @@ -105,6 +107,8 @@ const routeFor = (slug: string): string => { return "guardrails"; case "policies": return "policies"; + case "chat": + return "chat"; // tools case "mcp-servers": @@ -371,19 +375,29 @@ const Sidebar2: React.FC = ({ accessToken, userRole, defaultSelect }, [pathname, filteredMenuItems, defaultSelectedKey]); // ----- Navigation ----- - const goTo = (slug: string) => { + const goTo = (slug: string, newTab?: boolean) => { const href = toHref(slug); - router.push(href); + if (newTab) { + window.open(href, "_blank"); + } else { + router.push(href); + } }; // Wrap label in so every nav item supports right-click → "Open in new tab" // and Ctrl/Cmd+click to open in a new tab, while preserving SPA navigation for normal clicks. - const renderNavLink = (label: string, page: string): React.ReactNode => { + const renderNavLink = (label: string, page: string, newTab?: boolean): React.ReactNode => { const href = toHref(page); return ( { + if (newTab) { + e.stopPropagation(); + return; + } if (e.metaKey || e.ctrlKey || e.shiftKey || e.button === 1) { e.stopPropagation(); return; @@ -409,6 +423,8 @@ const Sidebar2: React.FC = ({ accessToken, userRole, defaultSelect style={{ transition: "all 0.3s cubic-bezier(0.4, 0, 0.2, 1)", position: "relative", + display: "flex", + flexDirection: "column", }} > = ({ accessToken, userRole, defaultSelect borderRight: 0, backgroundColor: "transparent", fontSize: "14px", + flex: 1, + overflowY: "auto", }} items={filteredMenuItems.map((item) => ({ key: item.key, icon: item.icon, - label: renderNavLink(item.label, item.page), + label: renderNavLink(item.label, item.page, item.newTab), children: item.children?.map((child) => ({ key: child.key, icon: child.icon, - label: renderNavLink(child.label, child.page), - onClick: () => goTo(child.page), + label: renderNavLink(child.label, child.page, child.newTab), + onClick: () => goTo(child.page, child.newTab), })), - onClick: !item.children ? () => goTo(item.page) : undefined, + onClick: !item.children ? () => goTo(item.page, item.newTab) : undefined, }))} /> {isAdminRole(userRole) && !collapsed && } + + {/* Pinned "Open Chat" button at bottom */} + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts index f81ade047e..681bf4161a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServerHealth.ts @@ -1,4 +1,5 @@ -import { useQuery } from "@tanstack/react-query"; +import { useCallback, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { fetchMCPServerHealth } from "@/components/networking"; import useAuthorized from "../useAuthorized"; @@ -12,11 +13,47 @@ interface MCPServerHealth { export const useMCPServerHealth = () => { const { accessToken } = useAuthorized(); - return useQuery({ + const queryClient = useQueryClient(); + const [recheckingServerIds, setRecheckingServerIds] = useState>(new Set()); + + const query = useQuery({ queryKey: mcpServerHealthKeys.lists(), queryFn: async () => await fetchMCPServerHealth(accessToken!), enabled: !!accessToken, // Refetch health status every 30 seconds to keep it up to date refetchInterval: 30000, }); + + const recheckServerHealth = useCallback(async (serverId: string) => { + if (!accessToken) return; + + setRecheckingServerIds((prev) => new Set(prev).add(serverId)); + + try { + const result: MCPServerHealth[] = await fetchMCPServerHealth(accessToken, [serverId]); + + queryClient.setQueriesData( + { queryKey: mcpServerHealthKeys.lists() }, + (oldData) => { + if (!oldData) return result; + return oldData.map((h) => { + const updated = result.find((r) => r.server_id === h.server_id); + return updated ?? h; + }); + }, + ); + } finally { + setRecheckingServerIds((prev) => { + const next = new Set(prev); + next.delete(serverId); + return next; + }); + } + }, [accessToken, queryClient]); + + return { + ...query, + recheckServerHealth, + recheckingServerIds, + }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts index 8746baae14..9210e25e1a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts @@ -6,11 +6,11 @@ import useAuthorized from "../useAuthorized"; const mcpServersKeys = createQueryKeys("mcpServers"); -export const useMCPServers = () => { +export const useMCPServers = (teamId?: string | null) => { const { accessToken } = useAuthorized(); return useQuery({ - queryKey: mcpServersKeys.list({}), - queryFn: async () => await fetchMCPServers(accessToken!), + queryKey: mcpServersKeys.list(teamId ? { filters: { teamId } } : undefined), + queryFn: async () => await fetchMCPServers(accessToken!, teamId), enabled: !!accessToken, }); }; diff --git a/ui/litellm-dashboard/src/components/chat/ChatMessages.tsx b/ui/litellm-dashboard/src/components/chat/ChatMessages.tsx index 10bf6b6192..2ff07c7065 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatMessages.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatMessages.tsx @@ -8,6 +8,7 @@ import remarkGfm from "remark-gfm"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; import ReasoningContent from "../playground/chat_ui/ReasoningContent"; +import MCPEventsDisplay from "../playground/chat_ui/MCPEventsDisplay"; import { ChatMessage } from "./types"; const { Panel } = Collapse; @@ -237,6 +238,8 @@ interface AssistantBubbleProps { isLastMessage: boolean; isStreaming: boolean; isTypingIndicator: boolean; + /** MCP events stored on the message — rendered inline below the response. */ + mcpEvents?: ChatMessage["mcpEvents"]; } function AssistantBubble({ @@ -244,6 +247,7 @@ function AssistantBubble({ isLastMessage, isStreaming, isTypingIndicator, + mcpEvents, }: AssistantBubbleProps) { // Ref to control ReasoningContent collapse on streaming end. // ReasoningContent manages its own expanded state; we use a key to @@ -321,6 +325,11 @@ function AssistantBubble({ + {mcpEvents && mcpEvents.length > 0 && ( +
+ +
+ )} ); } @@ -566,6 +575,7 @@ const ChatMessages: React.FC = ({ messages, isStreaming, onEditMessage }) isLastMessage={isLastMessage} isStreaming={isStreaming} isTypingIndicator={isLastMessage && isTypingIndicator} + mcpEvents={msg.mcpEvents} /> ); })} diff --git a/ui/litellm-dashboard/src/components/chat/ChatPage.tsx b/ui/litellm-dashboard/src/components/chat/ChatPage.tsx index 34473165bb..6a67814e2c 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatPage.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatPage.tsx @@ -26,6 +26,8 @@ import MCPConnectPicker from "./MCPConnectPicker"; import MCPAppsPanel from "./MCPAppsPanel"; import { fetchAvailableModels } from "../playground/llm_calls/fetch_models"; import { makeOpenAIChatCompletionRequest } from "../playground/llm_calls/chat_completion"; +import { makeOpenAIResponsesRequest } from "../playground/llm_calls/responses_api"; +import type { MCPEvent } from "./types"; import { getProxyBaseUrl } from "@/components/networking"; import { useUIConfig } from "@/app/(dashboard)/hooks/uiConfig/useUIConfig"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; @@ -135,6 +137,7 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user const [modelSearchText, setModelSearchText] = useState(""); const [selectedMCPServers, setSelectedMCPServers] = useState([]); + const [responsesSessionId, setResponsesSessionId] = useState(null); const [isStreaming, setIsStreaming] = useState(false); const [inputText, setInputText] = useState(""); const [mcpPopoverOpen, setMcpPopoverOpen] = useState(false); @@ -162,7 +165,7 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user createConversation, appendMessage, updateLastAssistantMessage, - truncateAfterMessage, + truncateFromMessage, deleteConversation, renameConversation, } = useChatHistory(activeConversationId); @@ -203,6 +206,12 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user if (staleId) router.replace(getChatUrl(uiRoot)); }, [staleId, router]); + // Reset the responses session when switching between conversations so that + // previous_response_id from conversation A is never sent for conversation B. + useEffect(() => { + setResponsesSessionId(null); + }, [activeConversationId]); + const toggleModel = useCallback((model: string) => { setSelectedModels((prev) => { let next: string[]; @@ -231,6 +240,7 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user let convId = activeConversationId; if (!convId) { convId = createConversation(model); + setResponsesSessionId(null); // new conversation starts a fresh session router.push(getChatUrl(uiRoot, convId)); } @@ -240,29 +250,56 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user setIsStreaming(true); abortControllerRef.current = new AbortController(); - const history = [ - ...(historyOverride ?? (activeConversation?.messages ?? []) - .filter((m) => m.role === "user" || m.role === "assistant") - .map((m) => ({ - role: m.role as "user" | "assistant", - content: m.content, - }))), - { role: "user" as const, content: trimmed }, - ]; + // When historyOverride is set (edit / retry), the existing server-side + // session chain covers messages that were just truncated and is no longer + // valid for the rewritten history. Eagerly clear the session so that a + // failed/aborted edit does not leave a stale session ID that contaminates + // the next regular send. + if (historyOverride) { + setResponsesSessionId(null); + } + + // On a normal continuation turn with an active session, the Responses API + // already holds the prior context server-side, so we only pass the new + // user message (sending the full history would double-count it). + // + // On the very first turn (no session yet), we send the full history. + const previousResponseId = historyOverride ? null : responsesSessionId; + + const history: Array<{ role: "user" | "assistant"; content: string }> = + historyOverride + ? [...historyOverride, { role: "user" as const, content: trimmed }] + : previousResponseId + ? [{ role: "user" as const, content: trimmed }] + : [ + // Explicitly filter to only user/assistant roles — tool messages + // lack a required tool_call_id and would cause API errors. + ...(activeConversation?.messages ?? []) + .filter((m): m is typeof m & { role: "user" | "assistant" } => + m.role === "user" || m.role === "assistant" + ) + .map((m) => ({ role: m.role, content: m.content })), + { role: "user" as const, content: trimmed }, + ]; let accumulatedContent = ""; let accumulatedReasoning = ""; + // MCP events accumulated locally so we can persist them to the message + // without relying on component state (which would cause stale closures). + const accumulatedMCPEvents: MCPEvent[] = []; + // Track clean completion so partial events are not shown on error/abort. + let streamCompletedCleanly = false; try { - await makeOpenAIChatCompletionRequest( + await makeOpenAIResponsesRequest( history, - (chunk: string) => { + (_role: string, chunk: string) => { accumulatedContent += chunk; updateLastAssistantMessage(convId!, { content: accumulatedContent }); }, model, accessToken, - undefined, + undefined, // tags abortControllerRef.current.signal, (rc: string) => { accumulatedReasoning += rc; @@ -270,7 +307,15 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user }, undefined, undefined, undefined, undefined, undefined, undefined, selectedMCPServers.length > 0 ? selectedMCPServers : undefined, + previousResponseId, + (id: string) => setResponsesSessionId(id), + (event: MCPEvent) => { + // Accumulate locally only — persisted once in finally to avoid + // one full localStorage write per MCP event during streaming. + accumulatedMCPEvents.push(event); + }, ); + streamCompletedCleanly = true; } catch (err: unknown) { if (err instanceof Error && err.name === "AbortError") { updateLastAssistantMessage(convId!, { @@ -282,12 +327,17 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user }); } } finally { + // Only persist MCP events on clean completion — partial events from an + // aborted or errored turn would show incomplete tool calls to the user. + if (accumulatedMCPEvents.length > 0 && streamCompletedCleanly) { + updateLastAssistantMessage(convId!, { mcpEvents: accumulatedMCPEvents }); + } setIsStreaming(false); abortControllerRef.current = null; } }, [activeConversationId, activeConversation, selectedModels, selectedMCPServers, accessToken, - createConversation, appendMessage, updateLastAssistantMessage, router, isStreaming], + createConversation, appendMessage, updateLastAssistantMessage, router, isStreaming, responsesSessionId], ); const handleSendComparison = useCallback( @@ -355,10 +405,10 @@ const ChatPage: React.FC = ({ accessToken, userRole, userId, user const priorMessages = (idx === -1 ? msgs : msgs.slice(0, idx)) .filter((m) => m.role === "user" || m.role === "assistant") .map((m) => ({ role: m.role as "user" | "assistant", content: m.content })); - truncateAfterMessage(activeConversationId, messageId); + truncateFromMessage(activeConversationId, messageId); handleSend(newContent, priorMessages); }, - [activeConversationId, isStreaming, activeConversation, truncateAfterMessage, handleSend], + [activeConversationId, isStreaming, activeConversation, truncateFromMessage, handleSend], ); const handleSubmit = useCallback( diff --git a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx index f6d728f788..625478e98a 100644 --- a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx +++ b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx @@ -170,9 +170,25 @@ const MCPAppsPanel: React.FC = ({ accessToken, selectedServers, onChange {/* Avatar + name + connect */}
+ {detailServer.mcp_info?.logo_url ? ( + {`${name} { + const el = e.target as HTMLImageElement; + el.style.display = "none"; + if (el.nextElementSibling) (el.nextElementSibling as HTMLElement).style.display = "flex"; + }} + /> + ) : null}
@@ -351,9 +367,26 @@ const MCPAppsPanel: React.FC = ({ accessToken, selectedServers, onChange onMouseEnter={(e) => { (e.currentTarget as HTMLDivElement).style.background = "#fafafa"; }} onMouseLeave={(e) => { (e.currentTarget as HTMLDivElement).style.background = "#fff"; }} > + {server.mcp_info?.logo_url ? ( + {`${name} { + const el = e.target as HTMLImageElement; + el.style.display = "none"; + if (el.nextElementSibling) (el.nextElementSibling as HTMLElement).style.display = "flex"; + }} + /> + ) : null}
{name.charAt(0).toUpperCase()} diff --git a/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx index 234aa8a528..a52ce18156 100644 --- a/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx +++ b/ui/litellm-dashboard/src/components/chat/MCPConnectPicker.tsx @@ -112,6 +112,18 @@ const MCPConnectPicker: React.FC = ({ accessToken, selectedServers, onCha gap: 12, }} > + {server.mcp_info?.logo_url && ( + {`${name} { (e.target as HTMLImageElement).style.display = "none"; }} + /> + )}
; toolResult?: string; diff --git a/ui/litellm-dashboard/src/components/chat/useChatHistory.ts b/ui/litellm-dashboard/src/components/chat/useChatHistory.ts index 1f27a15b39..b0a42d82de 100644 --- a/ui/litellm-dashboard/src/components/chat/useChatHistory.ts +++ b/ui/litellm-dashboard/src/components/chat/useChatHistory.ts @@ -51,8 +51,9 @@ export function useChatHistory(activeConversationId: string | null): { staleId: boolean; createConversation: (model: string) => string; appendMessage: (conversationId: string, message: Omit) => void; - updateLastAssistantMessage: (conversationId: string, updates: Partial>) => void; - truncateAfterMessage: (conversationId: string, messageId: string) => void; + updateLastAssistantMessage: (conversationId: string, updates: Partial>) => void; + /** Remove the message with `messageId` and all subsequent messages from the conversation. */ + truncateFromMessage: (conversationId: string, messageId: string) => void; deleteConversation: (id: string) => void; renameConversation: (id: string, newTitle: string) => void; setActiveConversationId: (id: string | null) => void; @@ -148,7 +149,7 @@ export function useChatHistory(activeConversationId: string | null): { const updateLastAssistantMessage = useCallback( ( conversationId: string, - updates: Partial>, + updates: Partial>, ) => { setConversations((prev) => { const updated = prev.map((conv) => { @@ -168,7 +169,7 @@ export function useChatHistory(activeConversationId: string | null): { [], ); - const truncateAfterMessage = useCallback( + const truncateFromMessage = useCallback( (conversationId: string, messageId: string) => { setConversations((prev) => { const updated = prev.map((conv) => { @@ -222,7 +223,7 @@ export function useChatHistory(activeConversationId: string | null): { createConversation, appendMessage, updateLastAssistantMessage, - truncateAfterMessage, + truncateFromMessage, deleteConversation, renameConversation, setActiveConversationId, diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx index d94a80e502..dc4ed25786 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx @@ -13,6 +13,7 @@ interface MCPServerSelectorProps { accessToken: string; placeholder?: string; disabled?: boolean; + teamId?: string | null; } const MCPServerSelector: React.FC = ({ @@ -22,8 +23,9 @@ const MCPServerSelector: React.FC = ({ accessToken, placeholder = "Select MCP servers", disabled = false, + teamId, }) => { - const { data: mcpServers = [], isLoading: serversLoading } = useMCPServers(); + const { data: mcpServers = [], isLoading: serversLoading } = useMCPServers(teamId); const { data: accessGroups = [], isLoading: groupsLoading } = useMCPAccessGroups(); const loading = serversLoading || groupsLoading; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPLogoSelector.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPLogoSelector.tsx new file mode 100644 index 0000000000..be05d74ec2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPLogoSelector.tsx @@ -0,0 +1,123 @@ +import React, { useState } from "react"; +import { Input, Tooltip } from "antd"; +import { InfoCircleOutlined, LinkOutlined } from "@ant-design/icons"; + +const logos = "/ui/assets/logos/"; + +const WELL_KNOWN_LOGOS: { name: string; url: string }[] = [ + { name: "GitHub", url: `${logos}github.svg` }, + { name: "Slack", url: `${logos}slack.svg` }, + { name: "Notion", url: `${logos}notion.svg` }, + { name: "Linear", url: `${logos}linear.svg` }, + { name: "Jira", url: `${logos}jira.svg` }, + { name: "Figma", url: `${logos}figma.svg` }, + { name: "Gmail", url: `${logos}gmail.svg` }, + { name: "Google Drive", url: `${logos}google_drive.svg` }, + { name: "Stripe", url: `${logos}stripe.svg` }, + { name: "Shopify", url: `${logos}shopify.svg` }, + { name: "Salesforce", url: `${logos}salesforce.svg` }, + { name: "HubSpot", url: `${logos}hubspot.svg` }, + { name: "Twilio", url: `${logos}twilio.svg` }, + { name: "Cloudflare", url: `${logos}cloudflare.svg` }, + { name: "Sentry", url: `${logos}sentry.svg` }, + { name: "PostgreSQL", url: `${logos}postgresql.svg` }, + { name: "Snowflake", url: `${logos}snowflake.svg` }, + { name: "Zapier", url: `${logos}zapier.svg` }, + { name: "Google", url: `${logos}google.svg` }, + { name: "GitLab", url: `${logos}gitlab.svg` }, +]; + +interface MCPLogoSelectorProps { + value?: string; + onChange?: (url: string | undefined) => void; +} + +const MCPLogoSelector: React.FC = ({ value, onChange }) => { + const [imgErrors, setImgErrors] = useState>(new Set()); + + const handleSelect = (url: string) => { + onChange?.(value === url ? undefined : url); + }; + + const handleImgError = (url: string) => { + setImgErrors((prev) => new Set(prev).add(url)); + }; + + return ( +
+
+ Logo + + + +
+ + {/* Preview */} + {value && ( +
+ Selected logo { (e.target as HTMLImageElement).style.display = "none"; }} + /> +
+
{value}
+
+ +
+ )} + + {/* Well-known logo grid */} +
+ {WELL_KNOWN_LOGOS.map((logo) => { + const isSelected = value === logo.url; + const hasFailed = imgErrors.has(logo.url); + if (hasFailed) return null; + return ( + + + + ); + })} +
+ + {/* Custom URL input */} + } + placeholder="Or paste a custom logo URL..." + value={value && !WELL_KNOWN_LOGOS.some((l) => l.url === value) ? value : ""} + onChange={(e) => { + const v = e.target.value.trim(); + onChange?.(v || undefined); + }} + className="rounded-lg" + size="small" + /> +
+ ); +}; + +export default MCPLogoSelector; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx b/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx index 23aae6cb14..b25e1dcadd 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/OpenAPIFormSection.tsx @@ -12,6 +12,8 @@ interface OpenAPIFormSectionProps { onValuesChange: (updates: Record) => void; /** Called when key tools change (from registry preset selection). */ onKeyToolsChange?: (tools: OpenAPIKeyTool[]) => void; + /** Called when a preset is selected so the parent can set the logo URL from icon_url. */ + onLogoUrlChange?: (url: string | undefined) => void; /** Called when the OAuth docs URL changes (e.g. link to create a GitHub OAuth App). */ onOAuthDocsUrlChange?: (url: string | null) => void; } @@ -26,6 +28,7 @@ const OpenAPIFormSection: React.FC = ({ accessToken, onValuesChange, onKeyToolsChange, + onLogoUrlChange, onOAuthDocsUrlChange, }) => { const [selectedPreset, setSelectedPreset] = useState(null); @@ -33,6 +36,7 @@ const OpenAPIFormSection: React.FC = ({ const handlePresetSelect = (entry: OpenAPIRegistryEntry) => { setSelectedPreset(entry.name); onKeyToolsChange?.(entry.key_tools ?? []); + onLogoUrlChange?.(entry.icon_url || undefined); const updates: Record = { spec_path: entry.spec_url, }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 90ecd4731c..74945731a2 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -11,6 +11,7 @@ import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; import MCPPermissionManagement from "./MCPPermissionManagement"; import OpenAPIFormSection, { OpenAPIKeyTool } from "./OpenAPIFormSection"; +import MCPLogoSelector from "./MCPLogoSelector"; import { isAdminRole } from "@/utils/roles"; import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "../molecules/notifications_manager"; @@ -70,6 +71,7 @@ const CreateMCPServer: React.FC = ({ const [keyTools, setKeyTools] = useState([]); const [searchValue, setSearchValue] = useState(""); const [oauthAccessToken, setOauthAccessToken] = useState(null); + const [logoUrl, setLogoUrl] = useState(undefined); const [oauthDocsUrl, setOauthDocsUrl] = useState(null); // Single hook call shared by MCPConnectionStatus and MCPToolConfiguration to avoid duplicate requests. @@ -101,6 +103,7 @@ const CreateMCPServer: React.FC = ({ allowedTools, searchValue, aliasManuallyEdited, + logoUrl, }), ); } catch (err) { @@ -202,6 +205,9 @@ const CreateMCPServer: React.FC = ({ if (typeof parsed.aliasManuallyEdited === "boolean") { setAliasManuallyEdited(parsed.aliasManuallyEdited); } + if (parsed.logoUrl) { + setLogoUrl(parsed.logoUrl); + } } catch (err) { console.error("Failed to restore MCP create state", err); } finally { @@ -357,6 +363,7 @@ const CreateMCPServer: React.FC = ({ mcp_info: { server_name: restValues.server_name || restValues.url, description: restValues.description, + logo_url: logoUrl || undefined, mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, }, mcp_access_groups: accessGroups, @@ -394,6 +401,7 @@ const CreateMCPServer: React.FC = ({ clearTools(); setAllowedTools([]); setAliasManuallyEdited(false); + setLogoUrl(undefined); setModalVisible(false); onCreateSuccess(response); } @@ -414,6 +422,7 @@ const CreateMCPServer: React.FC = ({ clearTools(); setAllowedTools([]); setAliasManuallyEdited(false); + setLogoUrl(undefined); setModalVisible(false); }; @@ -590,6 +599,8 @@ const CreateMCPServer: React.FC = ({ /> + + GitHub / Source URL} name="source_url" @@ -645,6 +656,7 @@ const CreateMCPServer: React.FC = ({ setFormValues((prev) => ({ ...prev, ...updates })) } onKeyToolsChange={setKeyTools} + onLogoUrlChange={setLogoUrl} onOAuthDocsUrlChange={setOauthDocsUrl} /> )} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx index 8130f96856..ea5ccf1c84 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_columns.tsx @@ -1,3 +1,4 @@ +import { useState } from "react"; import { ColumnDef } from "@tanstack/react-table"; import { MCPServer } from "./types"; import { Icon } from "@tremor/react"; @@ -6,6 +7,82 @@ import { getMaskedAndFullUrl } from "./utils"; import { Tooltip } from "antd"; import { CheckOutlined } from "@ant-design/icons"; +const HealthStatusBadge: React.FC<{ + server: MCPServer; + isLoadingHealth?: boolean; + isRechecking?: boolean; + onRecheck?: (serverId: string) => void; +}> = ({ server, isLoadingHealth, isRechecking, onRecheck }) => { + const [isHovered, setIsHovered] = useState(false); + const status = server.status || "unknown"; + const lastCheck = server.last_health_check; + const error = server.health_check_error; + + if (isLoadingHealth || isRechecking) { + return ( + + + Checking + + ); + } + + const getStatusColor = (status: string) => { + switch (status) { + case "healthy": + return "text-green-700 bg-green-50 border border-green-200"; + case "unhealthy": + return "text-red-700 bg-red-50 border border-red-200"; + default: + return "text-gray-600 bg-gray-50 border border-gray-200"; + } + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case "healthy": + return "✓"; + case "unhealthy": + return "✗"; + default: + return "?"; + } + }; + + const isClickable = !!onRecheck; + + const tooltipContent = ( +
+
Health Status: {status}
+ {lastCheck &&
Last Check: {new Date(lastCheck).toLocaleString()}
} + {error && ( +
+
Error:
+
{error}
+
+ )} + {!lastCheck && !error &&
No health check data available
} + {isClickable &&
Click to recheck
} +
+ ); + + return ( + + setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onClick={isClickable ? () => onRecheck(server.server_id) : undefined} + > + {isHovered && isClickable ? "↻" : getStatusIcon(status)} + {isHovered && isClickable + ? "Recheck" + : status.charAt(0).toUpperCase() + status.slice(1)} + + + ); +}; + export const mcpServerColumns = ( userRole: string, onView: (serverId: string) => void, @@ -13,6 +90,8 @@ export const mcpServerColumns = ( onDelete: (serverId: string) => void, isLoadingHealth?: boolean, onByokConnect?: (server: MCPServer) => void, + onRecheckHealth?: (serverId: string) => void, + recheckingServerIds?: Set, ): ColumnDef[] => [ { accessorKey: "server_id", @@ -31,6 +110,23 @@ export const mcpServerColumns = ( accessorKey: "server_name", header: "Name", enableSorting: true, + cell: ({ row }) => { + const logoUrl = row.original.mcp_info?.logo_url; + const name = row.original.server_name; + return ( +
+ {logoUrl ? ( + {`${name { (e.target as HTMLImageElement).style.display = "none"; }} + /> + ) : null} + {name} +
+ ); + }, }, { accessorKey: "alias", @@ -81,68 +177,14 @@ export const mcpServerColumns = ( { id: "health_status", header: "Health Status", - cell: ({ row }) => { - const server = row.original; - const status = server.status || "unknown"; - const lastCheck = server.last_health_check; - const error = server.health_check_error; - - if (isLoadingHealth) { - return ( - - - Checking - - ); - } - - const getStatusColor = (status: string) => { - switch (status) { - case "healthy": - return "text-green-700 bg-green-50 border border-green-200"; - case "unhealthy": - return "text-red-700 bg-red-50 border border-red-200"; - default: - return "text-gray-600 bg-gray-50 border border-gray-200"; - } - }; - - const getStatusIcon = (status: string) => { - switch (status) { - case "healthy": - return "✓"; - case "unhealthy": - return "✗"; - default: - return "?"; - } - }; - - const tooltipContent = ( -
-
Health Status: {status}
- {lastCheck &&
Last Check: {new Date(lastCheck).toLocaleString()}
} - {error && ( -
-
Error:
-
{error}
-
- )} - {!lastCheck && !error &&
No health check data available
} -
- ); - - return ( - - - {getStatusIcon(status)} - {status.charAt(0).toUpperCase() + status.slice(1)} - - - ); - }, + cell: ({ row }) => ( + + ), }, { id: "mcp_access_groups", diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index fc55542a0c..eadf93d8a9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -8,6 +8,7 @@ import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; +import MCPLogoSelector from "./MCPLogoSelector"; import { validateMCPServerUrl, validateMCPServerName } from "./utils"; import NotificationsManager from "../molecules/notifications_manager"; import { useMcpOAuthFlow } from "@/hooks/useMcpOAuthFlow"; @@ -41,6 +42,7 @@ const MCPServerEdit: React.FC = ({ const [toolNameToDisplayName, setToolNameToDisplayName] = useState>({}); const [toolNameToDescription, setToolNameToDescription] = useState>({}); const [pendingRestoredValues, setPendingRestoredValues] = useState | null>(null); + const [logoUrl, setLogoUrl] = useState(mcpServer.mcp_info?.logo_url || undefined); const authType = Form.useWatch("auth_type", form) as string | undefined; const transportType = Form.useWatch("transport", form) as string | undefined; const isStdioTransport = transportType === "stdio"; @@ -538,6 +540,7 @@ const MCPServerEdit: React.FC = ({ mcp_info: { server_name: mcpInfoServerName, description: restValues.description, + logo_url: logoUrl || undefined, mcp_server_cost_info: Object.keys(costConfig).length > 0 ? costConfig : null, }, mcp_access_groups: accessGroups, @@ -604,6 +607,7 @@ const MCPServerEdit: React.FC = ({ + @@ -818,6 +820,122 @@ const CreateMCPServer: React.FC = ({ /> )} + {transportType !== "stdio" && transportType !== "" && isAwsSigV4AuthType && ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[{ required: true, message: "AWS region is required for SigV4 auth" }]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + dependencies={[["credentials", "aws_secret_access_key"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const secretKey = getFieldValue(["credentials", "aws_secret_access_key"]); + if (secretKey && !value) { + return Promise.reject(new Error("Access Key ID is required when Secret Access Key is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + dependencies={[["credentials", "aws_access_key_id"]]} + rules={[ + ({ getFieldValue }) => ({ + validator(_, value) { + const accessKeyId = getFieldValue(["credentials", "aws_access_key_id"]); + if (accessKeyId && !value) { + return Promise.reject(new Error("Secret Access Key is required when Access Key ID is provided")); + } + return Promise.resolve(); + }, + }), + ]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + )} + {/* Stdio Configuration - only show for stdio transport */}
diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index eadf93d8a9..04cce34303 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -22,7 +22,7 @@ interface MCPServerEditProps { } const AUTH_TYPES_REQUIRING_AUTH_VALUE = [AUTH_TYPE.API_KEY, AUTH_TYPE.BEARER_TOKEN, AUTH_TYPE.TOKEN, AUTH_TYPE.BASIC]; -const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2]; +const AUTH_TYPES_REQUIRING_CREDENTIALS = [...AUTH_TYPES_REQUIRING_AUTH_VALUE, AUTH_TYPE.OAUTH2, AUTH_TYPE.AWS_SIGV4]; const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; const MCPServerEdit: React.FC = ({ @@ -50,6 +50,7 @@ const MCPServerEdit: React.FC = ({ const isMCPTransport = !isStdioTransport && !isOpenAPITransport; const shouldShowAuthValueField = authType ? AUTH_TYPES_REQUIRING_AUTH_VALUE.includes(authType) : false; const isOAuthAuthType = authType === AUTH_TYPE.OAUTH2; + const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4; const oauthFlowTypeValue = Form.useWatch("oauth_flow_type", form) as string | undefined; const isM2MFlow = isOAuthAuthType && oauthFlowTypeValue === OAUTH_FLOW.M2M; @@ -665,6 +666,7 @@ const MCPServerEdit: React.FC = ({ Token Basic Auth OAuth + AWS SigV4 (Bedrock AgentCore MCPs) )} @@ -883,6 +885,100 @@ const MCPServerEdit: React.FC = ({ )} + {!isStdioTransport && isAwsSigV4AuthType && ( + <> +

+ For MCP servers hosted on AWS Bedrock AgentCore.{" "} + + View docs → + +

+ + AWS Region + + + + + } + name={["credentials", "aws_region_name"]} + rules={[]} + > + + + + AWS Service Name + + + + + } + name={["credentials", "aws_service_name"]} + > + + + + AWS Access Key ID + + + + + } + name={["credentials", "aws_access_key_id"]} + rules={[]} + > + + + + AWS Secret Access Key + + + + + } + name={["credentials", "aws_secret_access_key"]} + rules={[]} + > + + + + AWS Session Token + + + + + } + name={["credentials", "aws_session_token"]} + > + + + + )} + {/* Permission Management / Access Control Section */}
{ pagination?: PaginationState; onPaginationChange?: OnChangeFn; enablePagination?: boolean; + onRowClick?: (row: TData) => void; } export function AllModelsDataTable({ @@ -41,6 +42,7 @@ export function AllModelsDataTable({ pagination, onPaginationChange, enablePagination = false, + onRowClick, }: AllModelsDataTableProps) { const [columnResizeMode] = React.useState("onChange"); const [columnSizing, setColumnSizing] = React.useState({}); @@ -174,7 +176,11 @@ export function AllModelsDataTable({ ) : tableInstance.getRowModel().rows.length > 0 ? ( tableInstance.getRowModel().rows.map((row) => ( - + onRowClick?.(row.original)} + > {row.getVisibleCells().map((cell) => ( { pagination?: PaginationState; onPaginationChange?: OnChangeFn; enablePagination?: boolean; + onRowClick?: (row: TData) => void; } export function ModelDataTable({ @@ -40,6 +41,7 @@ export function ModelDataTable({ pagination, onPaginationChange, enablePagination = false, + onRowClick, }: ModelDataTableProps) { const [sorting, setSorting] = React.useState(defaultSorting); const [columnResizeMode] = React.useState("onChange"); @@ -164,7 +166,11 @@ export function ModelDataTable({ ) : tableInstance.getRowModel().rows.length > 0 ? ( tableInstance.getRowModel().rows.map((row) => ( - + onRowClick?.(row.original)} + className={onRowClick ? "cursor-pointer hover:bg-gray-50" : ""} + > {row.getVisibleCells().map((cell) => ( void, expandedRows: Set, setExpandedRows: (expandedRows: Set) => void, + onDeleteClick?: (modelId: string) => void, ): ColumnDef[] => [ { header: () => Model ID, @@ -67,7 +68,10 @@ export const columns = ( ellipsis className="text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs cursor-pointer w-full block" style={{ fontSize: 14, padding: '1px 8px' }} - onClick={() => setSelectedModelId(model.model_info.id)} + onClick={(e) => { + e.stopPropagation(); + setSelectedModelId(model.model_info.id); + }} > {model.model_info.id} @@ -297,7 +301,10 @@ export const columns = ( size="xs" variant="light" className="font-mono text-blue-500 bg-blue-50 hover:bg-blue-100 text-xs font-normal px-2 py-0.5 text-left overflow-hidden truncate w-full" - onClick={() => setSelectedTeamId(model.model_info.team_id)} + onClick={(e: React.MouseEvent) => { + e.stopPropagation(); + setSelectedTeamId(model.model_info.team_id); + }} > {model.model_info.team_id.slice(0, 7)}... @@ -409,9 +416,10 @@ export const columns = ( { - if (canEditModel) { - setSelectedModelId(model.model_info.id); + onClick={(e) => { + e.stopPropagation(); + if (canEditModel && onDeleteClick) { + onDeleteClick(model.model_info.id); } }} className={!canEditModel ? "opacity-50 cursor-not-allowed" : "cursor-pointer hover:text-red-600"} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index dcbdd2f73e..ea3d5a1622 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -9038,9 +9038,7 @@ export const updateUiSettings = async (accessToken: string, settings: Record = ({ accessToken, isEmbedded setLoading(true); const _modelHubData = await modelHubPublicModelsCall(); console.log("ModelHubData:", _modelHubData); - setModelHubData(_modelHubData); + setModelHubData(Array.isArray(_modelHubData) ? _modelHubData : []); } catch (error) { console.error("There was an error fetching the public model data", error); setServiceStatus("Service unavailable"); @@ -150,7 +150,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded setAgentLoading(true); const _agentHubData = await agentHubPublicModelsCall(); console.log("AgentHubData:", _agentHubData); - setAgentHubData(_agentHubData); + setAgentHubData(Array.isArray(_agentHubData) ? _agentHubData : []); } catch (error) { console.error("There was an error fetching the public agent data", error); } finally { @@ -163,7 +163,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded setMcpLoading(true); const _mcpHubData = await mcpHubPublicServersCall(); console.log("MCPHubData:", _mcpHubData); - setMcpHubData(_mcpHubData); + setMcpHubData(Array.isArray(_mcpHubData) ? _mcpHubData : []); } catch (error) { console.error("There was an error fetching the public MCP server data", error); } finally { @@ -199,7 +199,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const getUniqueProviders = (data: ModelGroupInfo[]) => { const providers = new Set(); data.forEach((model) => { - model.providers.forEach((provider) => providers.add(provider)); + (model.providers ?? []).forEach((provider) => providers.add(provider)); }); return Array.from(providers); }; @@ -532,7 +532,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "providers", enableSorting: true, cell: ({ row }) => { - const providers = row.original.providers; + const providers = row.original.providers ?? []; return (
@@ -760,7 +760,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "description", enableSorting: false, cell: ({ row }) => { - const description = row.original.description; + const description = row.original.description ?? ""; const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; return ( @@ -897,7 +897,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "mcp_info.description", enableSorting: false, cell: ({ row }) => { - const description = row.original.mcp_info?.description || "-"; + const description = String(row.original.mcp_info?.description ?? "-"); const truncated = description.length > 80 ? description.substring(0, 80) + "..." : description; return ( @@ -912,7 +912,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded accessorKey: "url", enableSorting: false, cell: ({ row }) => { - const url = row.original.url; + const url = row.original.url ?? ""; const truncated = url.length > 40 ? url.substring(0, 40) + "..." : url; return ( @@ -1336,7 +1336,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Providers:
- {selectedModel.providers.map((provider) => { + {(selectedModel.providers ?? []).map((provider) => { const { logo } = getProviderLogoAndName(provider); return ( @@ -1460,7 +1460,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded )} {/* Supported OpenAI Parameters */} - {selectedModel.supported_openai_params && ( + {selectedModel.supported_openai_params && selectedModel.supported_openai_params.length > 0 && (
Supported OpenAI Parameters
@@ -1634,7 +1634,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Input Modes:
- {selectedAgent.defaultInputModes?.map((mode) => ( + {(selectedAgent.defaultInputModes ?? []).map((mode) => ( {mode} @@ -1644,7 +1644,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded
Output Modes:
- {selectedAgent.defaultOutputModes?.map((mode) => ( + {(selectedAgent.defaultOutputModes ?? []).map((mode) => ( {mode} From fa68d69bcfedb04e0d54e32f498069d0c4c32c32 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 12 Mar 2026 10:28:27 -0300 Subject: [PATCH 27/32] fix: restore _get_effort_level and is_model_gpt_5_4_plus_model (PR #23151) Independent fix (base: main) collaterally removed by PR #23276. Restores: - _get_effort_level() for extracting effort from string or dict - is_model_gpt_5_4_plus_model() classmethod - effective_effort usage in xhigh/tool-drop/sampling/temperature guards - Azure: _get_effort_level import and usage for dict reasoning_effort - Azure: gpt-5.4+ tool+reasoning drop logic --- .../llms/azure/chat/gpt_5_transformation.py | 24 ++++-- .../llms/openai/chat/gpt_5_transformation.py | 77 +++++++++++++++---- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index a8c5a14ea5..81c3dfded7 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -4,7 +4,10 @@ from typing import List import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.chat.gpt_5_transformation import ( + OpenAIGPT5Config, + _get_effort_level, +) from litellm.types.llms.openai import AllMessageValues from .gpt_transformation import AzureOpenAIConfig @@ -85,20 +88,21 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) + effective_effort = _get_effort_level(reasoning_effort_value) # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning supports_none = self._supports_reasoning_effort_level(model, "none") - if reasoning_effort_value == "none" and not supports_none: + if effective_effort == "none" and not supports_none: if litellm.drop_params is True or ( drop_params is not None and drop_params is True ): non_default_params = non_default_params.copy() optional_params = optional_params.copy() - if non_default_params.get("reasoning_effort") == "none": + if _get_effort_level(non_default_params.get("reasoning_effort")) == "none": non_default_params.pop("reasoning_effort") - if optional_params.get("reasoning_effort") == "none": + if _get_effort_level(optional_params.get("reasoning_effort")) == "none": optional_params.pop("reasoning_effort") else: raise UnsupportedParamsError( @@ -121,9 +125,19 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): ) # Only drop reasoning_effort='none' for models that don't support it - if result.get("reasoning_effort") == "none" and not supports_none: + result_effort = _get_effort_level(result.get("reasoning_effort")) + if result_effort == "none" and not supports_none: result.pop("reasoning_effort") + # Azure Chat Completions: gpt-5.4+ does not support tools + reasoning together. + # Drop reasoning_effort when both are present (OpenAI routes to Responses API; Azure does not). + if self.is_model_gpt_5_4_plus_model(model): + has_tools = bool( + non_default_params.get("tools") or optional_params.get("tools") + ) + if has_tools and result_effort not in (None, "none"): + result.pop("reasoning_effort", None) + return result def transform_request( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index beb76f3d80..f186bc6085 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -25,6 +25,22 @@ def _normalize_reasoning_effort_for_chat_completion( return None +def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]: + """Extract the effective effort level from reasoning_effort (string or dict). + + Use this for guards that compare effort level (e.g. xhigh validation, "none" checks). + Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly + treated as effort="none" for validation purposes. + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, dict) and "effort" in value: + return value["effort"] + return None + + class OpenAIGPT5Config(OpenAIGPTConfig): """Configuration for gpt-5 models including GPT-5-Codex variants. @@ -70,6 +86,19 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model_name = model.split("/")[-1] return model_name.startswith("gpt-5.4") + @classmethod + def is_model_gpt_5_4_plus_model(cls, model: str) -> bool: + """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro).""" + model_name = model.split("/")[-1] + if not model_name.startswith("gpt-5."): + return False + try: + version_str = model_name.replace("gpt-5.", "").split("-")[0] + major = version_str.split(".")[0] + return int(major) >= 4 + except (ValueError, IndexError): + return False + @classmethod def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool: """Check if the model supports a specific reasoning_effort level. @@ -150,21 +179,32 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Normalize reasoning_effort: chat completion API expects a string, not a dict - # (e.g. {'effort': 'high', 'summary': 'detailed'} -> 'high') + # Get raw reasoning_effort and effective effort level for all guards. + # Use effective_effort (extracted string) for xhigh validation, "none" checks, and + # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} + # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. raw_reasoning_effort = ( non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") ) - normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) - if raw_reasoning_effort is not None and normalized is not None: - if "reasoning_effort" in non_default_params: - non_default_params["reasoning_effort"] = normalized - if "reasoning_effort" in optional_params: - optional_params["reasoning_effort"] = normalized + effective_effort = _get_effort_level(raw_reasoning_effort) - reasoning_effort = normalized or raw_reasoning_effort - if reasoning_effort is not None and reasoning_effort == "xhigh": + # Normalize to string for Chat Completions API when dict has only "effort". + # Preserve full dict (e.g. {"effort": "high", "summary": "detailed"}) for Responses API. + if isinstance(raw_reasoning_effort, dict) and set(raw_reasoning_effort.keys()) <= {"effort"}: + normalized = _normalize_reasoning_effort_for_chat_completion(raw_reasoning_effort) + if normalized is not None: + if "reasoning_effort" in non_default_params: + non_default_params["reasoning_effort"] = normalized + if "reasoning_effort" in optional_params: + optional_params["reasoning_effort"] = normalized + + reasoning_effort = ( + non_default_params.get("reasoning_effort") + or optional_params.get("reasoning_effort") + or raw_reasoning_effort + ) + if effective_effort is not None and effective_effort == "xhigh": if not self._supports_reasoning_effort_level(model, "xhigh"): if litellm.drop_params or drop_params: non_default_params.pop("reasoning_effort", None) @@ -191,17 +231,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig): has_tools = bool( non_default_params.get("tools") or optional_params.get("tools") ) - if has_tools and reasoning_effort not in (None, "none"): - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - reasoning_effort = None + if has_tools and effective_effort not in (None, "none"): + # Check if this will be routed to Responses API + # If so, keep reasoning_effort; otherwise drop it for chat completions API + if not self.is_model_gpt_5_4_plus_model(model): + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + reasoning_effort = None # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none" supports_none = self._supports_reasoning_effort_level(model, "none") if supports_none: sampling_params = ["logprobs", "top_logprobs", "top_p"] has_sampling = any(p in non_default_params for p in sampling_params) - if has_sampling and reasoning_effort not in (None, "none"): + if has_sampling and effective_effort not in (None, "none"): if litellm.drop_params or drop_params: for p in sampling_params: non_default_params.pop(p, None) @@ -211,7 +254,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when " "reasoning_effort='none'. Current reasoning_effort='{}'. " "To drop unsupported params set `litellm.drop_params = True`" - ).format(reasoning_effort), + ).format(effective_effort), status_code=400, ) @@ -219,7 +262,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): temperature_value: Optional[float] = non_default_params.pop("temperature") if temperature_value is not None: # models supporting reasoning_effort="none" also support flexible temperature - if supports_none and (reasoning_effort == "none" or reasoning_effort is None): + if supports_none and (effective_effort == "none" or effective_effort is None): optional_params["temperature"] = temperature_value elif temperature_value == 1: optional_params["temperature"] = temperature_value From feed274aa3d6757f42f56473c06c985f1ccf9413 Mon Sep 17 00:00:00 2001 From: Chesars Date: Thu, 12 Mar 2026 13:36:57 -0300 Subject: [PATCH 28/32] Reapply "feat: add model_cost aliases expansion support" This reverts commit 3d2df7e8b5614927fec951fb8a5c0530b5e7f97c. --- litellm/caching/dual_cache.py | 4 + .../transformation.py | 13 + litellm/litellm_core_utils/duration_parser.py | 10 +- litellm/litellm_core_utils/redact_messages.py | 47 ++++ .../bedrock/chat/converse_transformation.py | 27 +- .../llms/fireworks_ai/chat/transformation.py | 5 +- litellm/llms/sagemaker/completion/handler.py | 56 ++-- .../llms/vertex_ai/gemini/transformation.py | 4 + .../proxy/_experimental/mcp_server/server.py | 11 +- litellm/proxy/auth/model_checks.py | 18 +- .../proxy/credential_endpoints/endpoints.py | 97 +++---- .../management_endpoints/team_endpoints.py | 18 -- .../pass_through_endpoints.py | 3 +- .../spend_management_endpoints.py | 20 +- .../transformation.py | 110 ++++++-- litellm/router.py | 22 ++ litellm/router_strategy/lowest_latency.py | 8 +- provider_endpoints_support.json | 18 ++ tests/llm_translation/test_skills_api.py | 16 +- .../test_custom_callback_input.py | 8 +- .../test_logging_redaction_e2e_test.py | 15 +- tests/test_litellm/caching/test_dual_cache.py | 103 +++++++ ...responses_transformation_transformation.py | 217 ++++++++++++++- .../chat/test_azure_gpt5_transformation.py | 17 ++ .../chat/test_converse_transformation.py | 50 ++++ .../test_fireworks_ai_chat_transformation.py | 54 ++++ .../chat/test_openai_gpt_transformation.py | 193 ++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 77 +++++- ...est_sagemaker_embedding_role_assumption.py | 243 +++++++++++++++++ .../test_vertex_ai_gemini_transformation.py | 69 +++++ .../mcp_server/test_mcp_server.py | 147 ++++++++++ .../proxy/auth/test_model_checks.py | 134 ++++++++++ .../test_pass_through_endpoints.py | 23 +- .../test_spend_tracking_utils.py | 5 +- .../proxy/test_openapi_schema_validation.py | 142 ++++++++++ .../test_litellm_completion_responses.py | 125 +++++++++ .../test_router_retry_non_retryable_errors.py | 251 ++++++++++++++++++ .../VirtualKeysPage/VirtualKeysTable.test.tsx | 82 +++++- .../VirtualKeysPage/VirtualKeysTable.tsx | 48 +++- 39 files changed, 2315 insertions(+), 195 deletions(-) create mode 100644 tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py create mode 100644 tests/test_litellm/proxy/test_openapi_schema_validation.py create mode 100644 tests/test_litellm/test_router_retry_non_retryable_errors.py diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 9bfcc411d4..4020b8cc22 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -348,6 +348,8 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: + if "ttl" not in kwargs and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache(key, value, **kwargs) if self.redis_cache is not None and local_only is False: @@ -369,6 +371,8 @@ class DualCache(BaseCache): ) try: if self.in_memory_cache is not None: + if "ttl" not in kwargs and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl await self.in_memory_cache.async_set_cache_pipeline( cache_list=cache_list, **kwargs ) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index e117fd528f..f1e9b18450 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -398,6 +398,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ResponseOutputMessage, ResponseReasoningItem, ) + from openai.types.responses.response_output_item import ResponseApplyPatchToolCall from litellm.types.utils import Choices, Message @@ -456,6 +457,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 + elif isinstance(item, ResponseApplyPatchToolCall): + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + tool_call_dict = LiteLLMCompletionResponsesConfig.convert_apply_patch_tool_call_to_chat_completion_tool_call( + tool_call_item=item, + index=tool_call_index, + ) + accumulated_tool_calls.append(tool_call_dict) + tool_call_index += 1 + elif isinstance(item, dict) and handle_raw_dict_callback is not None: # Handle raw dict responses (e.g., from GPT-5 Codex) choice, index = handle_raw_dict_callback(item=item, index=index) diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 70c28c4e06..6d2b4226ff 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -64,12 +64,10 @@ def duration_in_seconds(duration: str) -> int: now = time.time() current_time = datetime.fromtimestamp(now) - if current_time.month == 12: - target_year = current_time.year + 1 - target_month = 1 - else: - target_year = current_time.year - target_month = current_time.month + value + # Calculate target month and year, handling overflow past December + total_months = current_time.month - 1 + value # 0-indexed months + target_year = current_time.year + total_months // 12 + target_month = total_months % 12 + 1 # back to 1-indexed # Determine the day to set for next month target_day = current_time.day diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 9a5e4d183b..dbeb411107 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -75,6 +75,53 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, dict) and "choices" in response: + # ModelResponse dict format - redact content in choices + if isinstance(response.get("choices"), list): + for choice in response["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # For other formats (empty dict, None, etc.), use simple text format + standard_logging_object["response"] = {"text": redacted_str} + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 828fba66d2..13cd9aae93 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -51,6 +51,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ( ChatCompletionMessageToolCall, + CompletionTokensDetailsWrapper, Function, Message, ModelResponse, @@ -63,6 +64,7 @@ from litellm.utils import ( has_tool_call_blocks, last_assistant_with_tool_calls_has_no_thinking_blocks, supports_reasoning, + token_counter, ) from ..common_utils import ( @@ -1637,7 +1639,11 @@ class AmazonConverseConfig(BaseConfig): thinking_blocks_list.append(_redacted_block) return thinking_blocks_list - def _transform_usage(self, usage: ConverseTokenUsageBlock) -> Usage: + def _transform_usage( + self, + usage: ConverseTokenUsageBlock, + reasoning_content: Optional[str] = None, + ) -> Usage: input_tokens = usage["inputTokens"] output_tokens = usage["outputTokens"] total_tokens = usage["totalTokens"] @@ -1654,6 +1660,19 @@ class AmazonConverseConfig(BaseConfig): prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cache_read_input_tokens ) + reasoning_tokens = ( + token_counter(text=reasoning_content, count_response_tokens=True) + if reasoning_content + else 0 + ) + completion_tokens_details = CompletionTokensDetailsWrapper( + reasoning_tokens=reasoning_tokens, + text_tokens=( + output_tokens - reasoning_tokens + if reasoning_tokens > 0 + else output_tokens + ), + ) openai_usage = Usage( prompt_tokens=input_tokens, completion_tokens=output_tokens, @@ -1661,6 +1680,7 @@ class AmazonConverseConfig(BaseConfig): prompt_tokens_details=prompt_tokens_details, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, + completion_tokens_details=completion_tokens_details, ) return openai_usage @@ -1997,7 +2017,10 @@ class AmazonConverseConfig(BaseConfig): chat_completion_message["tool_calls"] = filtered_tools ## CALCULATING USAGE - bedrock returns usage in the headers - usage = self._transform_usage(completion_response["usage"]) + usage = self._transform_usage( + completion_response["usage"], + reasoning_content=chat_completion_message.get("reasoning_content"), + ) ## HANDLE TOOL CALLS _message = Message(**chat_completion_message) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 30f5536323..8407e8ab69 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -429,8 +429,11 @@ class FireworksAIConfig(OpenAIGPTConfig): "FIREWORKS_ACCOUNT_ID is not set. Please set the environment variable, to query Fireworks AI's `/models` endpoint." ) + base = api_base.rstrip("/") + if base.endswith("/v1"): + base = base[: -len("/v1")] response = litellm.module_level_client.get( - url=f"{api_base}/v1/accounts/{account_id}/models", + url=f"{base}/v1/accounts/{account_id}/models", headers={"Authorization": f"Bearer {api_key}"}, ) diff --git a/litellm/llms/sagemaker/completion/handler.py b/litellm/llms/sagemaker/completion/handler.py index 2a30dc5ef3..efbb218f57 100644 --- a/litellm/llms/sagemaker/completion/handler.py +++ b/litellm/llms/sagemaker/completion/handler.py @@ -583,35 +583,17 @@ class SagemakerLLM(BaseAWSLLM): ### BOTO3 INIT import boto3 - # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them - aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) - aws_access_key_id = optional_params.pop("aws_access_key_id", None) - aws_region_name = optional_params.pop("aws_region_name", None) + # Use _load_credentials to support role assumption (aws_role_name, aws_session_name) + credentials, aws_region_name = self._load_credentials(optional_params) - if aws_access_key_id is not None: - # uses auth params passed to completion - # aws_access_key_id is not None, assume user is trying to auth using litellm.completion - client = boto3.client( - service_name="sagemaker-runtime", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - region_name=aws_region_name, - ) - else: - # aws_access_key_id is None, assume user is trying to auth using env variables - # boto3 automaticaly reads env variables - - # we need to read region name from env - # I assume majority of users use .env for auth - region_name = ( - get_secret("AWS_REGION_NAME") - or aws_region_name # get region from config file if specified - or "us-west-2" # default to us-west-2 if region not specified - ) - client = boto3.client( - service_name="sagemaker-runtime", - region_name=region_name, - ) + # Create boto3 session with the loaded credentials + session = boto3.Session( + aws_access_key_id=credentials.access_key, + aws_secret_access_key=credentials.secret_key, + aws_session_token=credentials.token, + region_name=aws_region_name, + ) + client = session.client(service_name="sagemaker-runtime") # pop streaming if it's in the optional params as 'stream' raises an error with sagemaker inference_params = deepcopy(optional_params) @@ -628,7 +610,9 @@ class SagemakerLLM(BaseAWSLLM): #### EMBEDDING LOGIC # Transform request based on model type provider_config = SagemakerEmbeddingConfig.get_model_config(model) - request_data = provider_config.transform_embedding_request(model, input, optional_params, {}) + request_data = provider_config.transform_embedding_request( + model, input, optional_params, {} + ) data = json.dumps(request_data).encode("utf-8") ## LOGGING @@ -673,19 +657,19 @@ class SagemakerLLM(BaseAWSLLM): ) print_verbose(f"raw model_response: {response}") - + # Transform response based on model type from httpx import Response as HttpxResponse - + # Create a mock httpx Response object for the transformation mock_response = HttpxResponse( status_code=200, - content=json.dumps(response).encode('utf-8'), - headers={"content-type": "application/json"} + content=json.dumps(response).encode("utf-8"), + headers={"content-type": "application/json"}, ) - + model_response = EmbeddingResponse() - + # Use the request_data that was already transformed above return provider_config.transform_embedding_response( model=model, @@ -695,5 +679,5 @@ class SagemakerLLM(BaseAWSLLM): api_key=None, request_data=request_data, optional_params=optional_params, - litellm_params=litellm_params or {} + litellm_params=litellm_params or {}, ) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 7bfde06fd8..cbb7c7172f 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -593,6 +593,10 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 raise e +# Keys that LiteLLM consumes internally and must never be forwarded to the +_LITELLM_INTERNAL_EXTRA_BODY_KEYS: frozenset = frozenset({"cache", "tags"}) + + def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: """Pop extra_body from optional_params and shallow-merge into data, deep-merging dict values.""" extra_body: Optional[dict] = optional_params.pop("extra_body", None) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index fd777b81b2..0a2fe332c7 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -718,6 +718,7 @@ if MCP_AVAILABLE: Checks both the full tool name and unprefixed version (without server prefix). This allows users to configure simple tool names regardless of prefixing. + Comparison is case-insensitive to handle OpenAPI operationIds that may be in camelCase. Args: tool_name: The tool name to check (may be prefixed like "server-tool_name") @@ -730,13 +731,15 @@ if MCP_AVAILABLE: split_server_prefix_from_name, ) - # Check if the full name is in the list - if tool_name in filter_list: + # Normalize filter list to lowercase for case-insensitive comparison + filter_list_lower = [f.lower() for f in filter_list] + + if tool_name.lower() in filter_list_lower: return True - # Check if the unprefixed name is in the list + # Check if the unprefixed name is in the list (case-insensitive) unprefixed_name, _ = split_server_prefix_from_name(tool_name) - return unprefixed_name in filter_list + return unprefixed_name.lower() in filter_list_lower def filter_tools_by_allowed_tools( tools: List[MCPTool], diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 82b13760d7..d988579b91 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -111,12 +111,19 @@ def get_key_models( if SpecialModelNames.all_team_models.value in all_models: all_models = list(user_api_key_dict.team_models) # copy to avoid mutating cached objects if SpecialModelNames.all_proxy_models.value in all_models: - all_models = proxy_model_list + all_models = list(proxy_model_list) # copy to avoid mutating caller's list + if include_model_access_groups: + all_models.extend(model_access_groups.keys()) all_models = _get_models_from_access_groups( - model_access_groups=model_access_groups, all_models=all_models + model_access_groups=model_access_groups, + all_models=all_models, + include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) return all_models @@ -140,8 +147,8 @@ def get_team_models( all_models_set.update(team_models) if SpecialModelNames.all_proxy_models.value in all_models_set: all_models_set.update(proxy_model_list) - - all_models = list(all_models_set) + if include_model_access_groups: + all_models_set.update(model_access_groups.keys()) all_models = _get_models_from_access_groups( model_access_groups=model_access_groups, @@ -149,6 +156,9 @@ def get_team_models( include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL TEAM MODELS - {}".format(len(all_models))) return all_models diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 9f228bb118..5fa9546e00 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -142,17 +142,47 @@ async def get_credentials( tags=["credential management"], response_model=CredentialItem, ) +async def get_credential_by_name( + request: Request, + fastapi_response: Response, + credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + [BETA] endpoint. This might change unexpectedly. + """ + try: + for credential in litellm.credential_list: + if credential.credential_name == credential_name: + masked_credential = CredentialItem( + credential_name=credential.credential_name, + credential_values=_get_masked_values( + credential.credential_values, + unmasked_length=4, + number_of_asterisks=4, + ), + credential_info=credential.credential_info, + ) + return masked_credential + raise HTTPException( + status_code=404, + detail="Credential not found. Got credential name: " + credential_name, + ) + except Exception as e: + verbose_proxy_logger.exception(e) + raise handle_exception_on_proxy(e) + + @router.get( "/credentials/by_model/{model_id}", dependencies=[Depends(user_api_key_auth)], tags=["credential management"], response_model=CredentialItem, ) -async def get_credential( +async def get_credential_by_model( request: Request, fastapi_response: Response, - credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"), - model_id: Optional[str] = None, + model_id: str = Path(..., description="The model ID to look up credentials for"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -161,48 +191,25 @@ async def get_credential( from litellm.proxy.proxy_server import llm_router try: - if model_id: - if llm_router is None: - raise HTTPException(status_code=500, detail="LLM router not found") - model = llm_router.get_deployment(model_id) - if model is None: - raise HTTPException(status_code=404, detail="Model not found") - credential_values = llm_router.get_deployment_credentials(model_id) - if credential_values is None: - raise HTTPException(status_code=404, detail="Model not found") - masked_credential_values = _get_masked_values( - credential_values, - unmasked_length=4, - number_of_asterisks=4, - ) - credential = CredentialItem( - credential_name="{}-credential-{}".format(model.model_name, model_id), - credential_values=masked_credential_values, - credential_info={}, - ) - # return credential object - return credential - elif credential_name: - for credential in litellm.credential_list: - if credential.credential_name == credential_name: - masked_credential = CredentialItem( - credential_name=credential.credential_name, - credential_values=_get_masked_values( - credential.credential_values, - unmasked_length=4, - number_of_asterisks=4, - ), - credential_info=credential.credential_info, - ) - return masked_credential - raise HTTPException( - status_code=404, - detail="Credential not found. Got credential name: " + credential_name, - ) - else: - raise HTTPException( - status_code=404, detail="Credential name or model ID required" - ) + if llm_router is None: + raise HTTPException(status_code=500, detail="LLM router not found") + model = llm_router.get_deployment(model_id) + if model is None: + raise HTTPException(status_code=404, detail="Model not found") + credential_values = llm_router.get_deployment_credentials(model_id) + if credential_values is None: + raise HTTPException(status_code=404, detail="Model not found") + masked_credential_values = _get_masked_values( + credential_values, + unmasked_length=4, + number_of_asterisks=4, + ) + credential = CredentialItem( + credential_name="{}-credential-{}".format(model.model_name, model_id), + credential_values=masked_credential_values, + credential_info={}, + ) + return credential except Exception as e: verbose_proxy_logger.exception(e) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 633de86aa6..ee1868fc74 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2827,21 +2827,6 @@ async def validate_membership( ) -def _unfurl_all_proxy_models( - team_info: LiteLLM_TeamTable, llm_router: Router -) -> LiteLLM_TeamTable: - if ( - SpecialModelNames.all_proxy_models.value in team_info.models - and llm_router is not None - ): - team_models: set[str] = set() # make set to avoid duplicates - for model in team_info.models: - if model != SpecialModelNames.all_proxy_models.value: - team_models.add(model) - for model in llm_router.get_model_names(): - team_models.add(model) - team_info.models = list(team_models) - return team_info async def _add_team_member_budget_table( @@ -2972,9 +2957,6 @@ async def team_info( team_info_response_object=_team_info, ) - # ## UNFURL 'all-proxy-models' into the team_info.models list ## - # if llm_router is not None: - # _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index a51d5e82b0..bc103a3fc4 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -2059,7 +2059,8 @@ class InitPassThroughEndpointHelpers: """ ## CHECK IF MAPPED PASS THROUGH ENDPOINT for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: - if route.startswith(mapped_route): + full_mapped_route = InitPassThroughEndpointHelpers._build_full_path_with_root(mapped_route) + if route.startswith(full_mapped_route): return True # Fast path: check if any registered route key contains this path diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 4da6ff7be2..b3b4b55af1 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1461,11 +1461,21 @@ async def _get_spend_report_for_time_range( dependencies=[Depends(user_api_key_auth)], responses={ 200: { - "cost": { - "description": "The calculated cost", - "example": 0.0, - "type": "float", - } + "description": "The calculated cost", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "cost": { + "type": "number", + "description": "The calculated cost", + "example": 0.0, + } + }, + } + } + }, } }, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index f72c4e46d7..8e5dd2bd06 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -292,21 +292,21 @@ class LiteLLMCompletionResponsesConfig: ) _messages = litellm_completion_request.get("messages") or [] session_messages = chat_completion_session.get("messages") or [] - + # If session messages are empty (e.g., no database in test environment), # we still need to process the new input messages # Store original _messages before combining for safety check original_new_messages = _messages.copy() if _messages else [] - + combined_messages = session_messages + _messages - + # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache tools = litellm_completion_request.get("tools") or [] combined_messages = LiteLLMCompletionResponsesConfig._ensure_tool_results_have_corresponding_tool_calls( messages=combined_messages, tools=tools ) - + # Safety check: Ensure we don't end up with empty messages # This can happen when using previous_response_id without a database (e.g., in tests) # and session messages are empty but new input messages exist @@ -340,7 +340,7 @@ class LiteLLMCompletionResponsesConfig: "custom_llm_provider", "" ), ) - + litellm_completion_request["messages"] = combined_messages litellm_completion_request["litellm_trace_id"] = chat_completion_session.get( "litellm_session_id" @@ -386,10 +386,45 @@ class LiteLLMCompletionResponsesConfig: if call_id_raw: existing_tool_call_ids.add(str(call_id_raw)) + ######################################################### + # Merge consecutive function_call items into a single assistant + # message. Anthropic requires that all tool_use blocks appear in + # ONE assistant message immediately followed by the tool_result + # blocks. Without this merging, each function_call creates its own + # assistant message, producing back-to-back assistant messages that + # Anthropic rejects with "tool_use ids were found without + # tool_result blocks immediately after". + ######################################################### + if messages: + last_msg = messages[-1] + last_role = ( + last_msg.get("role") + if isinstance(last_msg, dict) + else getattr(last_msg, "role", None) + ) + if last_role == "assistant": + for new_msg in chat_completion_messages: + new_role = ( + new_msg.get("role") + if isinstance(new_msg, dict) + else getattr(new_msg, "role", None) + ) + if new_role == "assistant": + new_tcs = ( + new_msg.get("tool_calls") + if isinstance(new_msg, dict) + else getattr(new_msg, "tool_calls", None) + ) or [] + for tc in new_tcs: + LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( + last_msg, tc + ) + continue + ######################################################### # If Input Item is a Tool Call Output, add it to the tool_call_output_messages list - # preserving the ordering of tool call outputs. Some models require the tool - # result to immediately follow the assistant tool call. + # preserving the ordering of tool call outputs. Some models require the tool + # result to immediately follow the assistant tool call. ######################################################### if LiteLLMCompletionResponsesConfig._is_input_item_tool_call_output( input_item=_input @@ -774,14 +809,14 @@ class LiteLLMCompletionResponsesConfig: ]: """ Ensure that tool_result messages have corresponding tool_calls in the previous assistant message. - + This is critical for Anthropic API which requires that each tool_result block has a corresponding tool_use block in the previous assistant message. - + Args: messages: List of messages that may include tool_result messages tools: Optional list of tools that can be used to reconstruct tool_calls if not in cache - + Returns: List of messages with tool_calls added to assistant messages when needed """ @@ -801,18 +836,18 @@ class LiteLLMCompletionResponsesConfig: ] ] = list(copy.deepcopy(messages)) messages_to_remove = [] - + # Count non-tool messages to avoid removing all messages # This prevents empty messages list when using previous_response_id without a database non_tool_messages_count = sum( 1 for msg in fixed_messages if msg.get("role") != "tool" ) - + for i, message in enumerate(fixed_messages): # Only process tool messages - check role first to narrow the type if message.get("role") != "tool": continue - + # At this point, we know it's a tool message, so it should have tool_call_id # Use get() with default to safely access tool_call_id tool_call_id_raw = ( @@ -823,11 +858,11 @@ class LiteLLMCompletionResponsesConfig: tool_call_id: str = ( str(tool_call_id_raw) if tool_call_id_raw is not None else "" ) - + prev_assistant_idx = LiteLLMCompletionResponsesConfig._find_previous_assistant_idx( fixed_messages, i ) - + # Try to recover empty tool_call_id from previous assistant message if not tool_call_id and prev_assistant_idx is not None: prev_assistant = fixed_messages[prev_assistant_idx] @@ -842,7 +877,7 @@ class LiteLLMCompletionResponsesConfig: message_dict["tool_call_id"] = tool_call_id elif hasattr(message, "tool_call_id"): setattr(message, "tool_call_id", tool_call_id) - + # Only remove messages with empty tool_call_id if we have other non-tool messages # This prevents ending up with an empty messages list when using previous_response_id # without a database (e.g., in tests where session messages are empty) @@ -854,7 +889,7 @@ class LiteLLMCompletionResponsesConfig: # If no non-tool messages, keep the tool message even with empty call_id # The API will return a proper error message about the missing tool_use block continue - + # Check if the previous assistant message has the corresponding tool_call # This needs to run for ALL tool messages with a valid tool_call_id, # not just those that had an empty tool_call_id initially @@ -863,12 +898,12 @@ class LiteLLMCompletionResponsesConfig: tool_calls = LiteLLMCompletionResponsesConfig._get_tool_calls_list( prev_assistant ) - + if not LiteLLMCompletionResponsesConfig._check_tool_call_exists( tool_calls, tool_call_id ): _tool_use_definition = TOOL_CALLS_CACHE.get_cache(key=tool_call_id) - + if not _tool_use_definition and tools: _tool_use_definition = LiteLLMCompletionResponsesConfig._reconstruct_tool_call_from_tools( tool_call_id, tools @@ -891,11 +926,11 @@ class LiteLLMCompletionResponsesConfig: LiteLLMCompletionResponsesConfig._add_tool_call_to_assistant( prev_assistant, tool_call_chunk ) - + # Remove messages with empty tool_call_id that couldn't be fixed for idx in reversed(messages_to_remove): fixed_messages.pop(idx) - + return fixed_messages @staticmethod @@ -1547,6 +1582,39 @@ class LiteLLMCompletionResponsesConfig: return tool_call_dict + @staticmethod + def convert_apply_patch_tool_call_to_chat_completion_tool_call( + tool_call_item: Any, + index: int = 0, + ) -> Dict[str, Any]: + """ + Convert ResponseApplyPatchToolCall to ChatCompletionToolCallChunk format. + + The operation (create_file / update_file / delete_file) is serialised + as JSON so it appears in function.arguments, just like any other + tool call. + + Args: + tool_call_item: ResponseApplyPatchToolCall object with call_id and operation + index: The index of this tool call + + Returns: + Dictionary in ChatCompletionToolCallChunk format + """ + import json + + operation_dict = tool_call_item.operation.model_dump() + tool_call_dict: Dict[str, Any] = { + "id": tool_call_item.call_id, + "function": { + "name": "apply_patch", + "arguments": json.dumps(operation_dict), + }, + "type": "function", + "index": index, + } + return tool_call_dict + @staticmethod def transform_chat_completion_response_to_responses_api_response( request_input: Union[str, ResponseInputParam], diff --git a/litellm/router.py b/litellm/router.py index 06def6ceb4..ecda6f4ab6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5505,6 +5505,10 @@ class Router: return response except Exception as e: + # Always track the latest error so we raise the most + # recent exception instead of the first one. + original_exception = e + ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt - 1 @@ -5519,6 +5523,24 @@ class Router: ) else: _healthy_deployments = [] + + # Check if this error is non-retryable (e.g., 400 context + # window exceeded). If so, raise immediately instead of + # continuing the retry loop. Respect retry policy + # precedence - only check when no retry policy applies. + if not _retry_policy_applies: + try: + self.should_retry_this_error( + error=e, + healthy_deployments=_healthy_deployments, + all_deployments=_all_deployments, + context_window_fallbacks=context_window_fallbacks, + regular_fallbacks=fallbacks, + content_policy_fallbacks=content_policy_fallbacks, + ) + except Exception: + raise e + _timeout = self._time_to_sleep_before_retry( e=e, remaining_retries=remaining_retries, diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index fbe8830946..20db28fa10 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -498,20 +498,22 @@ class LowestLatencyLoggingHandler(CustomLogger): # get average latency or average ttft (depending on streaming/non-streaming) total: float = 0.0 - if ( + use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 - ): + ) + if use_ttft: for _call_latency in item_ttft_latency: if isinstance(_call_latency, float): total += _call_latency + item_latency = total / len(item_ttft_latency) else: for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency - item_latency = total / len(item_latency) + item_latency = total / len(item_latency) # -------------- # # Debugging Logic diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { diff --git a/tests/llm_translation/test_skills_api.py b/tests/llm_translation/test_skills_api.py index 57b153cef0..76eb274293 100644 --- a/tests/llm_translation/test_skills_api.py +++ b/tests/llm_translation/test_skills_api.py @@ -44,19 +44,25 @@ def create_skill_zip(skill_name: str, unique_suffix: Optional[str] = None): skill_dir = test_dir / skill_name # Create a zip file containing the skill directory + # When unique_suffix is set, folder name must match skill name in SKILL.md (Anthropic requirement) + zip_folder_name = f"{skill_name}-{unique_suffix}" if unique_suffix else skill_name zip_path = test_dir / f"{skill_name}.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: - zf.write(skill_dir, arcname=skill_name) - if unique_suffix is not None: - # Rewrite SKILL.md with a unique name to avoid API conflicts + # Rewrite SKILL.md with a unique name and use matching folder name skill_md = (skill_dir / "SKILL.md").read_text() skill_md = skill_md.replace( f"name: {skill_name}", - f"name: {skill_name}-{unique_suffix}", + f"name: {zip_folder_name}", ) - zf.writestr(f"{skill_name}/SKILL.md", skill_md) + zf.writestr(f"{zip_folder_name}/SKILL.md", skill_md) + # Add any other files in the skill dir (e.g. subdirs) under the new folder name + for f in skill_dir.rglob("*"): + if f.is_file() and f.name != "SKILL.md": + rel = f.relative_to(skill_dir) + zf.write(f, arcname=f"{zip_folder_name}/{rel}") else: + zf.write(skill_dir, arcname=skill_name) zf.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md") try: diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index fcdfcfe6e7..ead387599d 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1300,9 +1300,11 @@ def test_logging_async_cache_hit_sync_call(turn_off_message_logging): "redacted-by-litellm" == standard_logging_object["messages"][0]["content"] ) - assert {"text": "redacted-by-litellm"} == standard_logging_object[ - "response" - ] + # response is a full ModelResponse dict (choices format) since d84e5e381acf + assert ( + standard_logging_object["response"]["choices"][0]["message"]["content"] + == "redacted-by-litellm" + ) def test_logging_standard_payload_failure_call(): diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 0536ec7205..0391a5a895 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -45,7 +45,8 @@ async def test_global_redaction_on(): await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload", @@ -75,7 +76,8 @@ async def test_global_redaction_with_dynamic_params(turn_off_message_logging): ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -108,7 +110,8 @@ async def test_global_redaction_off_with_dynamic_params(turn_off_message_logging json.dumps(standard_logging_payload, indent=2), ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -390,7 +393,8 @@ async def test_redaction_with_streaming_response(): assert standard_logging_payload is not None # Verify that redaction worked without pickle errors - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload for streaming with coroutine handling", @@ -477,5 +481,6 @@ async def test_redaction_with_metadata_completion_api(): # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, # the system checks the appropriate field for headers - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" diff --git a/tests/test_litellm/caching/test_dual_cache.py b/tests/test_litellm/caching/test_dual_cache.py index 9974c23e4b..606f25ddf4 100644 --- a/tests/test_litellm/caching/test_dual_cache.py +++ b/tests/test_litellm/caching/test_dual_cache.py @@ -1,9 +1,11 @@ import asyncio +import time from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache @@ -56,3 +58,104 @@ async def test_dual_cache_async_batch_get_cache_rolls_back_redis_reservation_on_ assert mock_async_batch_get_cache.call_count == 2 assert "shared_a" not in dual_cache.last_redis_batch_access_time assert "shared_b" not in dual_cache.last_redis_batch_access_time + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_injects_default_in_memory_ttl(): + """ + Test that async_set_cache injects default_in_memory_ttl into kwargs + when no explicit ttl is provided, matching the sync set_cache behavior. + + Regression test for: async_set_cache was missing the TTL injection that + sync set_cache has, causing InMemoryCache to use its own default_ttl (600s) + instead of DualCache's default_in_memory_ttl. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + before = time.time() + await dual_cache.async_set_cache(key="test_key", value="test_value") + after = time.time() + + # The TTL stored should reflect default_in_memory_ttl (60s), not + # InMemoryCache's default_ttl (600s) + expiry = in_memory_cache.ttl_dict["test_key"] + assert expiry >= before + 60 + assert expiry <= after + 60 + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_respects_explicit_ttl(): + """ + Test that async_set_cache does NOT override an explicitly provided ttl. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + before = time.time() + await dual_cache.async_set_cache(key="test_key", value="test_value", ttl=30) + after = time.time() + + # The explicit ttl=30 should be used, not default_in_memory_ttl (60) + expiry = in_memory_cache.ttl_dict["test_key"] + assert expiry >= before + 30 + assert expiry <= after + 30 + + +@pytest.mark.asyncio +async def test_dual_cache_async_set_cache_pipeline_injects_default_in_memory_ttl(): + """ + Test that async_set_cache_pipeline injects default_in_memory_ttl into kwargs + when no explicit ttl is provided. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + default_in_memory_ttl=60, + ) + + cache_list = [("key_a", "value_a"), ("key_b", "value_b")] + + before = time.time() + await dual_cache.async_set_cache_pipeline(cache_list=cache_list) + after = time.time() + + for key in ["key_a", "key_b"]: + expiry = in_memory_cache.ttl_dict[key] + assert expiry >= before + 60 + assert expiry <= after + 60 + + +@pytest.mark.asyncio +async def test_dual_cache_sync_and_async_set_cache_use_same_ttl(): + """ + Test that sync set_cache and async async_set_cache produce the same TTL + when no explicit ttl is provided, ensuring parity between the two paths. + """ + in_memory_sync = InMemoryCache(default_ttl=600) + dual_cache_sync = DualCache( + in_memory_cache=in_memory_sync, + default_in_memory_ttl=60, + ) + + in_memory_async = InMemoryCache(default_ttl=600) + dual_cache_async = DualCache( + in_memory_cache=in_memory_async, + default_in_memory_ttl=60, + ) + + dual_cache_sync.set_cache(key="test_key", value="test_value") + await dual_cache_async.async_set_cache(key="test_key", value="test_value") + + sync_expiry = in_memory_sync.ttl_dict["test_key"] + async_expiry = in_memory_async.ttl_dict["test_key"] + + # Both should use default_in_memory_ttl=60, so their expiry times + # should be within a small tolerance of each other + assert abs(sync_expiry - async_expiry) < 1.0 diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index ef3d7534d9..8c72b7725a 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -738,7 +738,58 @@ def test_response_completed_with_message_only_emits_stop_finish_reason(): ) -def test_function_call_done_does_not_emit_finish_reason(): + +def test_response_completed_preserves_usage_with_cached_tokens(): + """ + Test that response.completed correctly translates Responses API usage + (input_tokens_details) to chat completion usage (prompt_tokens_details). + + This is a regression test for an issue where streaming with models that + use the Responses API bridge (e.g. gpt-5.2-codex) would drop + prompt_tokens_details, causing cached_tokens to always be None. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + + chunk = { + "type": "response.completed", + "response": { + "id": "resp_789", + "status": "completed", + "output": [ + { + "type": "message", + "id": "msg_abc", + "role": "assistant", + "content": [{"type": "output_text", "text": "Six"}], + "status": "completed", + } + ], + "usage": { + "input_tokens": 1226, + "output_tokens": 5, + "total_tokens": 1231, + "input_tokens_details": {"cached_tokens": 1024}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.usage is not None, "usage should be set on response.completed chunk" + assert result.usage.prompt_tokens == 1226, "prompt_tokens should map from input_tokens" + assert result.usage.completion_tokens == 5, "completion_tokens should map from output_tokens" + assert result.usage.prompt_tokens_details is not None, "prompt_tokens_details should be set" + assert result.usage.prompt_tokens_details.cached_tokens == 1024, ( + "cached_tokens should be preserved from input_tokens_details" + ) + + +def test_function_call_done_emits_is_finished(): """ Test that OUTPUT_ITEM_DONE for a function_call does NOT emit finish_reason. The response.completed event handles the terminal finish_reason correctly. @@ -1327,6 +1378,138 @@ def test_transform_response_preserves_annotations(): print("✓ Annotations from Responses API are correctly preserved in Chat Completions format") +def test_apply_patch_tool_call_converted_to_chat_completion_tool_call(): + """ + Test that ResponseApplyPatchToolCall items from the Responses API are + correctly converted to ChatCompletions-style tool calls by the bridge. + + This is a regression test for a bug where litellm.completion() with a + responses/ model prefix crashed when the model returned an + apply_patch_call, because _convert_response_output_to_choices did not + handle ResponseApplyPatchToolCall items. The model DID use the tool, + but the bridge silently dropped it (or raised an error), while the + native litellm.responses() path worked correctly. + """ + import json + from unittest.mock import Mock + + from openai.types.responses.response_apply_patch_tool_call import ( + OperationCreateFile, + ) + from openai.types.responses.response_output_item import ( + ResponseApplyPatchToolCall, + ) + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ( + InputTokensDetails, + OutputTokensDetails, + ResponseAPIUsage, + ResponsesAPIResponse, + ) + from litellm.types.utils import ModelResponse, Usage + + handler = LiteLLMResponsesTransformationHandler() + + # Build an apply_patch_call item like the model would return + operation = OperationCreateFile( + diff="--- /dev/null\n+++ b/hello.py\n@@ -0,0 +1 @@\n+print('hello world')\n", + path="hello.py", + type="create_file", + ) + apply_patch_item = ResponseApplyPatchToolCall( + id="apc_001", + call_id="call_patch_hello", + operation=operation, + status="completed", + type="apply_patch_call", + ) + + # Minimal usage + usage = ResponseAPIUsage( + input_tokens=30, + input_tokens_details=InputTokensDetails(cached_tokens=0), + output_tokens=40, + output_tokens_details=OutputTokensDetails(reasoning_tokens=0), + total_tokens=70, + ) + + raw_response = ResponsesAPIResponse( + id="resp_apply_patch_test", + created_at=1234567890, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model="gpt-5.2-codex", + object="response", + output=[apply_patch_item], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + max_output_tokens=None, + previous_response_id=None, + reasoning=None, + status="completed", + text=None, + truncation="disabled", + usage=usage, + user=None, + store=True, + background=False, + ) + + model_response = ModelResponse( + id="chatcmpl-apply-patch", + created=1234567890, + model=None, + object="chat.completion", + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + + logging_obj = Mock() + + result = handler.transform_response( + model="gpt-5.2-codex", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.2-codex"}, + messages=[ + {"role": "system", "content": "You are a coding assistant."}, + {"role": "user", "content": "Create hello.py"}, + ], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + # Should have exactly one choice with finish_reason="tool_calls" + assert len(result.choices) == 1, f"Expected 1 choice, got {len(result.choices)}" + + choice = result.choices[0] + assert choice.finish_reason == "tool_calls" + + # The choice should contain one tool call for apply_patch + tool_calls = choice.message.tool_calls + assert tool_calls is not None, "tool_calls should not be None" + assert len(tool_calls) == 1, f"Expected 1 tool_call, got {len(tool_calls)}" + + tc = tool_calls[0] + assert tc["id"] == "call_patch_hello" + assert tc["type"] == "function" + assert tc["function"]["name"] == "apply_patch" + + # The operation should be serialised as JSON in arguments + args = json.loads(tc["function"]["arguments"]) + assert args["type"] == "create_file" + assert args["path"] == "hello.py" + assert "print('hello world')" in args["diff"] def test_multi_tool_call_stream_no_premature_finish(): """ Regression test for multi-tool-call streaming bug. @@ -1778,3 +1961,35 @@ def test_parallel_tool_calls_comprehensive_streaming_integration(): ) print("✓ Parallel tool calls with split argument deltas stream correctly end-to-end") + + +def test_map_optional_params_preserves_reasoning_summary(): + """Test that reasoning_effort dict with summary field is preserved. + + Regression test for: User reported that summary field was being dropped + when routing to Responses API. The dict format should be fully preserved. + """ + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams + + handler = LiteLLMResponsesTransformationHandler() + + optional_params = { + "stream": False, + "tools": [{"type": "function", "function": {"name": "test_tool"}}], + "tool_choice": "auto", + "reasoning_effort": {"effort": "high", "summary": "detailed"}, + } + + responses_api_request = ResponsesAPIOptionalRequestParams() + handler._map_optional_params_to_responses_api_request( + optional_params, responses_api_request + ) + + # Verify reasoning_effort dict with summary was fully preserved + assert "reasoning" in responses_api_request + assert responses_api_request["reasoning"] == {"effort": "high", "summary": "detailed"} + assert responses_api_request["reasoning"]["effort"] == "high" + assert responses_api_request["reasoning"]["summary"] == "detailed" diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 25f3d1364f..635359563b 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -192,6 +192,23 @@ def test_azure_gpt5_1_series_temperature_handling(config: AzureOpenAIGPT5Config) assert params["temperature"] == 0.6 +def test_azure_gpt5_4_drops_reasoning_effort_when_tools_present(config: AzureOpenAIGPT5Config): + """Azure Chat Completions: gpt-5.4+ drops reasoning_effort when tools are present. + + OpenAI routes tools+reasoning to Responses API; Azure does not, so we drop reasoning_effort. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": "high", "tools": tools}, + optional_params={}, + model="gpt5_series/gpt-5.4", + drop_params=False, + api_version="2024-05-01-preview", + ) + assert "reasoning_effort" not in params + assert params["tools"] == tools + + def test_azure_gpt5_reasoning_effort_none_error(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5 (non-5.1) raises error for reasoning_effort='none' when drop_params=False.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 345f3ae7c5..9892a0403b 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -43,6 +43,29 @@ def test_transform_usage(): ) assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] + # completion_tokens_details should always be populated + assert openai_usage.completion_tokens_details is not None + assert openai_usage.completion_tokens_details.reasoning_tokens == 0 + assert openai_usage.completion_tokens_details.text_tokens == usage["outputTokens"] + + +def test_transform_usage_with_reasoning_content(): + """Test that completion_tokens_details correctly tracks reasoning vs text tokens.""" + usage = ConverseTokenUsageBlock( + **{ + "inputTokens": 10, + "outputTokens": 100, + "totalTokens": 110, + } + ) + config = AmazonConverseConfig() + reasoning_text = "Let me think about this step by step." + openai_usage = config._transform_usage(usage, reasoning_content=reasoning_text) + assert openai_usage.completion_tokens_details is not None + assert openai_usage.completion_tokens_details.reasoning_tokens > 0 + assert openai_usage.completion_tokens_details.text_tokens == ( + usage["outputTokens"] - openai_usage.completion_tokens_details.reasoning_tokens + ) def test_transform_system_message(): @@ -3170,6 +3193,33 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" +def test_output_config_snake_case_stripped_from_bedrock_converse_request(): + """Test that output_config (snake_case) is stripped from Bedrock Converse requests. + + Bedrock Converse API doesn't support the output_config parameter (Anthropic-only). + Nova and other Converse models reject requests with extraneous output_config. + """ + config = AmazonConverseConfig() + messages = [{"role": "user", "content": "test"}] + optional_params = { + "output_config": {"effort": "high"}, + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # output_config must not appear in additionalModelRequestFields + additional = result.get("additionalModelRequestFields", {}) + assert "output_config" not in additional, ( + f"output_config should be stripped for Bedrock Converse, got: {list(additional.keys())}" + ) + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { diff --git a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 8006ffdff1..5d5aaa64c8 100644 --- a/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -110,6 +110,60 @@ def test_get_supported_openai_params_reasoning_effort(): assert "reasoning_effort" not in unsupported_params +@pytest.mark.parametrize( + "api_base, expected_url_prefix", + [ + ( + "https://api.fireworks.ai/inference/v1", + "https://api.fireworks.ai/inference/v1/accounts/", + ), + ( + "https://api.fireworks.ai/inference/v1/", + "https://api.fireworks.ai/inference/v1/accounts/", + ), + ( + "https://custom-host.example.com/v1", + "https://custom-host.example.com/v1/accounts/", + ), + ( + "https://custom-host.example.com/api", + "https://custom-host.example.com/api/v1/accounts/", + ), + ], + ids=["default", "trailing-slash", "custom-with-v1", "custom-without-v1"], +) +def test_get_models_url_no_double_v1(api_base, expected_url_prefix): + """Ensure get_models never produces a /v1/v1/ URL segment (fixes #23106).""" + config = FireworksAIConfig() + account_id = "fireworks" + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "models": [{"name": "accounts/fireworks/models/llama-v3-70b"}] + } + + with ( + patch("litellm.module_level_client.get", return_value=mock_response) as mock_get, + patch( + "litellm.llms.fireworks_ai.chat.transformation.get_secret_str", + side_effect=lambda key: { + "FIREWORKS_API_KEY": "test-key", + "FIREWORKS_API_BASE": api_base, + "FIREWORKS_ACCOUNT_ID": account_id, + }.get(key), + ), + ): + result = config.get_models(api_key="test-key", api_base=api_base) + + called_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args[1].get("url", "") + assert "/v1/v1/" not in called_url, f"Double /v1/ detected in URL: {called_url}" + assert called_url.startswith(expected_url_prefix), ( + f"URL {called_url} does not start with {expected_url_prefix}" + ) + assert result == ["fireworks_ai/accounts/fireworks/models/llama-v3-70b"] + + def test_transform_messages_helper_removes_provider_specific_fields(): """ Test that _transform_messages_helper removes provider_specific_fields from messages. diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index 39ff0a4f4d..90fdc2d20d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -13,6 +13,7 @@ from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config class TestOpenAIGPTConfig: @@ -324,3 +325,195 @@ class TestPromptCacheParams: ) assert optional_params.get("prompt_cache_key") == "my-cache-key" assert optional_params.get("prompt_cache_retention") == "24h" + + +class TestGPT5ReasoningEffortPreservation: + """Tests for GPT-5 reasoning_effort dict preservation for Responses API.""" + + def setup_method(self): + self.config = OpenAIGPT5Config() + + def test_reasoning_effort_string_preserved(self): + """Test that reasoning_effort as string is preserved.""" + non_default_params = {"reasoning_effort": "high"} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # String format should be preserved + assert non_default_params.get("reasoning_effort") == "high" + + def test_reasoning_effort_dict_with_only_effort_normalized(self): + """Test that reasoning_effort dict with only 'effort' key is normalized to string.""" + non_default_params = {"reasoning_effort": {"effort": "high"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with only 'effort' should be normalized to string + assert non_default_params.get("reasoning_effort") == "high" + + def test_reasoning_effort_dict_with_summary_preserved(self): + """Test that reasoning_effort dict with 'summary' field is preserved for Responses API. + + Regression test for: User reported that summary field was being dropped when + routing to Responses API. The dict format with additional fields should be + preserved so it can be properly handled by the Responses API transformation. + """ + non_default_params = {"reasoning_effort": {"effort": "high", "summary": "detailed"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with additional fields should be preserved + assert non_default_params.get("reasoning_effort") == {"effort": "high", "summary": "detailed"} + assert isinstance(non_default_params.get("reasoning_effort"), dict) + assert non_default_params["reasoning_effort"]["effort"] == "high" + assert non_default_params["reasoning_effort"]["summary"] == "detailed" + + def test_reasoning_effort_dict_with_generate_summary_preserved(self): + """Test that reasoning_effort dict with 'generate_summary' field is preserved.""" + non_default_params = {"reasoning_effort": {"effort": "medium", "generate_summary": "auto"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with additional fields should be preserved + assert non_default_params.get("reasoning_effort") == {"effort": "medium", "generate_summary": "auto"} + assert isinstance(non_default_params.get("reasoning_effort"), dict) + + def test_reasoning_effort_dict_with_all_fields_preserved(self): + """Test that reasoning_effort dict with all fields is preserved.""" + non_default_params = { + "reasoning_effort": { + "effort": "high", + "summary": "detailed", + "generate_summary": "concise" + } + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + # Dict with all fields should be preserved + reasoning = non_default_params.get("reasoning_effort") + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "high" + assert reasoning["summary"] == "detailed" + assert reasoning["generate_summary"] == "concise" + + def test_reasoning_effort_dict_xhigh_triggers_validation(self): + """xhigh-dict: effective effort is extracted for model-support validation. + + When reasoning_effort={"effort": "xhigh", "summary": "detailed"} is passed to a model + that doesn't support xhigh (e.g. gpt-5.1), the xhigh guard must fire. + """ + import litellm + + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + with pytest.raises(litellm.utils.UnsupportedParamsError): + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + def test_reasoning_effort_dict_xhigh_dropped_when_requested(self): + """xhigh-dict with drop_params=True: reasoning_effort is dropped.""" + non_default_params = {"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=True, + ) + + assert "reasoning_effort" not in non_default_params + + def test_reasoning_effort_dict_none_treated_as_none_for_tools(self): + """none-dict: {"effort": "none", "summary": "detailed"} is treated as effort=none. + + Tool-drop guard should NOT fire; reasoning_effort should be kept. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + non_default_params = {"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools} + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.4", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("tools") == tools + + def test_reasoning_effort_dict_none_treated_as_none_for_sampling(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows logprobs/top_p. + + Sampling-param guard should NOT fire; logprobs should be kept. + """ + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} + assert non_default_params.get("logprobs") is True + + def test_reasoning_effort_dict_none_allows_temperature(self): + """none-dict: {"effort": "none", "summary": "detailed"} allows non-default temperature.""" + non_default_params = { + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "temperature": 0.5, + } + optional_params = {} + + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="gpt-5.1", + drop_params=False, + ) + + assert optional_params.get("temperature") == 0.5 + assert non_default_params.get("reasoning_effort") == {"effort": "none", "summary": "detailed"} diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index b136f8774b..13d2ebab14 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -324,10 +324,11 @@ def test_gpt5_4_pro_allows_reasoning_effort_xhigh(config: OpenAIConfig): assert params["reasoning_effort"] == "xhigh" -def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): - """Chat completion API expects reasoning_effort as a string, not a dict. +def test_gpt5_preserves_reasoning_effort_dict_with_summary(config: OpenAIConfig): + """Dict with summary/generate_summary is preserved for Responses API. Config/deployments may pass Responses API format: {'effort': 'high', 'summary': 'detailed'}. + We preserve the full dict so it reaches the Responses API transformation. """ params = config.map_openai_params( non_default_params={"reasoning_effort": {"effort": "high", "summary": "detailed"}}, @@ -335,18 +336,82 @@ def test_gpt5_normalizes_reasoning_effort_dict_to_string(config: OpenAIConfig): model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "high" + assert params["reasoning_effort"] == {"effort": "high", "summary": "detailed"} -def test_gpt5_normalizes_reasoning_effort_dict_from_optional_params(config: OpenAIConfig): - """reasoning_effort dict in optional_params (e.g. from model config) is normalized.""" +def test_gpt5_xhigh_dict_triggers_validation(config: OpenAIConfig): + """Dict with effort='xhigh' triggers xhigh model-support validation. + + Regression: when reasoning_effort is a dict, effective_effort must be used for + the xhigh guard so validation is not silently skipped. + """ + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + + +def test_gpt5_xhigh_dict_accepted_for_supported_model(config: OpenAIConfig): + """Dict with effort='xhigh' passes through for gpt-5.4+.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "xhigh", "summary": "detailed"}}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "xhigh", "summary": "detailed"} + + +def test_gpt5_none_dict_with_tools_no_tool_drop(config: OpenAIConfig): + """Dict with effort='none' and tools: no tool-drop, reasoning_effort preserved. + + Regression: effective_effort='none' must be used for tool-drop guard so + {"effort": "none", "summary": "detailed"} is not incorrectly treated as non-none. + """ + tools = [{"type": "function", "function": {"name": "test", "description": "test"}}] + params = config.map_openai_params( + non_default_params={"reasoning_effort": {"effort": "none", "summary": "detailed"}, "tools": tools}, + optional_params={}, + model="gpt-5.4", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["tools"] == tools + + +def test_gpt5_none_dict_with_sampling_params_allowed(config: OpenAIConfig): + """Dict with effort='none' allows logprobs/top_p/top_logprobs. + + Regression: effective_effort='none' must be used for sampling guard so + {"effort": "none", "summary": "detailed"} does not incorrectly trigger sampling errors. + """ + params = config.map_openai_params( + non_default_params={ + "reasoning_effort": {"effort": "none", "summary": "detailed"}, + "logprobs": True, + "top_p": 0.9, + }, + optional_params={}, + model="gpt-5.1", + drop_params=False, + ) + assert params["reasoning_effort"] == {"effort": "none", "summary": "detailed"} + assert params["logprobs"] is True + assert params["top_p"] == 0.9 + + +def test_gpt5_preserves_reasoning_effort_dict_with_summary_from_optional_params(config: OpenAIConfig): + """reasoning_effort dict with summary in optional_params is preserved.""" params = config.map_openai_params( non_default_params={}, optional_params={"reasoning_effort": {"effort": "medium", "summary": "detailed"}}, model="gpt-5.4", drop_params=False, ) - assert params["reasoning_effort"] == "medium" + assert params["reasoning_effort"] == {"effort": "medium", "summary": "detailed"} def test_gpt5_4_drops_reasoning_effort_when_tools_present(config: OpenAIConfig): diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py new file mode 100644 index 0000000000..82c84af5e2 --- /dev/null +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_embedding_role_assumption.py @@ -0,0 +1,243 @@ +""" +Test cases for SageMaker embedding role assumption support + +This module tests that the SageMaker embedding handler properly supports +AWS IAM role assumption via aws_role_name and aws_session_name parameters, +matching the behavior of the completion handler. +""" + +import json +import os +import sys +from datetime import timezone +from unittest.mock import MagicMock, call, patch + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from botocore.credentials import Credentials + +from litellm.llms.sagemaker.completion.handler import SagemakerLLM +from litellm.types.utils import EmbeddingResponse + + +class TestSagemakerEmbeddingRoleAssumption: + """Test that SageMaker embedding supports role assumption like completion does""" + + def setup_method(self): + self.sagemaker_llm = SagemakerLLM() + + def test_embedding_uses_load_credentials(self): + """ + Test that embedding() calls _load_credentials() to support role assumption. + This ensures aws_role_name and aws_session_name parameters are properly handled. + """ + # Mock credentials that would be returned after role assumption + mock_credentials = Credentials( + access_key="assumed-access-key", + secret_key="assumed-secret-key", + token="assumed-session-token", + ) + + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + # Mock boto3.Session to return our mock client + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ) as mock_load_creds, patch("boto3.Session", return_value=mock_session): + + # Create mock logging object + mock_logging = MagicMock() + + optional_params = { + "aws_role_name": "arn:aws:iam::123456789012:role/TestRole", + "aws_session_name": "test-session", + } + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Verify _load_credentials was called with the optional_params + mock_load_creds.assert_called_once() + + # Verify boto3.Session was created with the assumed credentials + mock_session_calls = mock_session.client.call_args_list + assert len(mock_session_calls) == 1 + assert mock_session_calls[0] == call(service_name="sagemaker-runtime") + + def test_embedding_role_assumption_with_sts(self): + """ + Test the full role assumption flow for embeddings, similar to completion. + Verifies that STS assume_role is called when aws_role_name is provided. + """ + # Mock the STS client for role assumption + mock_sts_client = MagicMock() + + # Mock the STS response with proper expiration handling + mock_expiry = MagicMock() + mock_expiry.tzinfo = timezone.utc + time_diff = MagicMock() + time_diff.total_seconds.return_value = 3600 + mock_expiry.__sub__ = MagicMock(return_value=time_diff) + + mock_sts_response = { + "Credentials": { + "AccessKeyId": "assumed-access-key", + "SecretAccessKey": "assumed-secret-key", + "SessionToken": "assumed-session-token", + "Expiration": mock_expiry, + } + } + mock_sts_client.assume_role.return_value = mock_sts_response + + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + # Mock boto3.Session for SageMaker client creation + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + def mock_boto3_client(service_name, **kwargs): + if service_name == "sts": + return mock_sts_client + return mock_sagemaker_client + + with patch("boto3.client", side_effect=mock_boto3_client), \ + patch("boto3.Session", return_value=mock_session): + + mock_logging = MagicMock() + + optional_params = { + "aws_role_name": "arn:aws:iam::123456789012:role/CrossAccountRole", + "aws_session_name": "litellm-embedding-session", + "aws_region_name": "us-east-1", + } + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Verify STS assume_role was called with correct parameters + mock_sts_client.assume_role.assert_called_once() + call_args = mock_sts_client.assume_role.call_args + assert call_args[1]["RoleArn"] == "arn:aws:iam::123456789012:role/CrossAccountRole" + assert call_args[1]["RoleSessionName"] == "litellm-embedding-session" + + def test_embedding_without_role_assumption(self): + """ + Test that embedding works without role assumption when aws_role_name is not provided. + Should use default credentials from environment/instance profile. + """ + # Mock the SageMaker client response + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + + # Mock credentials returned from environment + mock_credentials = Credentials( + access_key="env-access-key", + secret_key="env-secret-key", + token=None, + ) + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-west-2") + ), patch("boto3.Session", return_value=mock_session): + + mock_logging = MagicMock() + + # No aws_role_name provided + optional_params = { + "aws_region_name": "us-west-2", + } + + result = self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params=optional_params, + ) + + # Should still work and return embeddings + assert result is not None + + def test_embedding_session_created_with_assumed_credentials(self): + """ + Test that boto3.Session is created with the credentials from role assumption. + This verifies the credentials flow from _load_credentials to the SageMaker client. + """ + mock_credentials = Credentials( + access_key="assumed-key", + secret_key="assumed-secret", + token="assumed-token", + ) + + mock_sagemaker_client = MagicMock() + mock_sagemaker_client.invoke_endpoint.return_value = { + "Body": MagicMock( + read=MagicMock(return_value=json.dumps({"embedding": [[0.1, 0.2, 0.3]]}).encode()) + ) + } + + with patch.object( + self.sagemaker_llm, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), patch("boto3.Session") as mock_session_class: + + mock_session = MagicMock() + mock_session.client.return_value = mock_sagemaker_client + mock_session_class.return_value = mock_session + + mock_logging = MagicMock() + + self.sagemaker_llm.embedding( + model="test-endpoint", + input=["hello world"], + model_response=EmbeddingResponse(), + print_verbose=print, + encoding=None, + logging_obj=mock_logging, + optional_params={}, + ) + + # Verify Session was created with the assumed credentials + mock_session_class.assert_called_once_with( + aws_access_key_id="assumed-key", + aws_secret_access_key="assumed-secret", + aws_session_token="assumed-token", + region_name="us-east-1", + ) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 444125dffa..ce3d2daa74 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -128,6 +128,75 @@ def test_vertex_ai_includes_labels(): +def test_extra_body_cache_not_forwarded_to_vertex_ai(): + """ + 'cache' inside extra_body is a LiteLLM-internal proxy caching control. + It must NOT be forwarded to the Vertex AI request body. + + Regression test for: "Invalid JSON payload received. Unknown name \"cache\": Cannot find field." + Vertex AI enforces a strict JSON schema and rejects any unknown field. + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "cache": {"use-cache": True, "ttl": 86400}, # LiteLLM-internal + "some_vertex_param": "value", # legitimate provider extra + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + # 'cache' must be stripped — Vertex AI has no such field + assert "cache" not in result, ( + "extra_body.cache must not be forwarded to Vertex AI. " + "Vertex AI rejects it with 400: Unknown name \"cache\": Cannot find field." + ) + + # Other legitimate extra_body keys should still pass through + assert "some_vertex_param" in result + assert result["some_vertex_param"] == "value" + + # Core request fields must be present + assert "contents" in result + + +def test_extra_body_tags_not_forwarded_to_vertex_ai(): + """ + 'tags' inside extra_body is a LiteLLM-internal param for logging/tracking. + It must NOT be forwarded to the Vertex AI request body. + Documented in litellm_proxy.md: "Send tags by including them in the extra_body parameter" + """ + messages = [{"role": "user", "content": "test"}] + optional_params = { + "extra_body": { + "tags": ["user:alice", "env:prod"], + "custom_param": "allowed", + }, + } + litellm_params = {} + + result = _transform_request_body( + messages=messages, + model="gemini-2.5-pro", + optional_params=optional_params, + custom_llm_provider="vertex_ai", + litellm_params=litellm_params, + cached_content=None, + ) + + assert "tags" not in result + assert "custom_param" in result + assert result["custom_param"] == "allowed" + + def test_metadata_to_labels_vertex_only(): """Test that metadata->labels conversion only happens for Vertex AI""" messages = [{"role": "user", "content": "test"}] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index de2ec13b4a..a104ac2257 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2093,3 +2093,150 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["tool_count_total"] == 1 assert spend_meta["allowed_server_count"] == 1 assert spend_meta["per_server_tool_counts"]["server_a"] == 1 + + +def test_tool_name_matches_case_insensitive(): + """Test that _tool_name_matches performs case-insensitive comparison. + + This is critical for OpenAPI-based MCP servers where: + 1. operationIds are often in camelCase (e.g., 'addPet', 'updatePet') + 2. Tool names are lowercased during registration (e.g., 'addpet', 'updatepet') + 3. allowed_tools configuration may use the original camelCase names + + Without case-insensitive matching, all tools would be filtered out. + """ + try: + from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + except ImportError: + pytest.skip("MCP server not available") + + # Test case 1: Unprefixed tool name with camelCase in filter list + assert _tool_name_matches("addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("deletepet", ["addPet", "updatePet"]) is False + + # Test case 2: Prefixed tool name with camelCase in filter list + assert _tool_name_matches("per_store-addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-deletepet", ["addPet", "updatePet"]) is False + + # Test case 3: Mixed case variations + assert _tool_name_matches("findPetsByStatus", ["findpetsbystatus"]) is True + assert _tool_name_matches("findpetsbystatus", ["findPetsByStatus"]) is True + assert _tool_name_matches("FINDPETSBYSTATUS", ["findPetsByStatus"]) is True + + # Test case 4: Full prefixed name in filter list (case-insensitive) + assert _tool_name_matches("server-addPet", ["server-addpet"]) is True + assert _tool_name_matches("server-addpet", ["server-addPet"]) is True + + # Test case 5: Ensure non-matching names still don't match + assert _tool_name_matches("addpet", ["deletePet", "updatePet"]) is False + assert _tool_name_matches("server-addpet", ["deletePet", "updatePet"]) is False + + +def test_filter_tools_by_allowed_tools_case_insensitive(): + """Test that filter_tools_by_allowed_tools handles case-insensitive matching. + + Ensures that OpenAPI tools with lowercase names can be filtered using + camelCase allowed_tools configuration from the OpenAPI spec. + """ + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + # Create mock tools with lowercase names (as registered from OpenAPI) + tools = [ + MCPTool( + name="per_store-addpet", + description="Add a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-updatepet", + description="Update a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-deletepet", + description="Delete a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-findpetsbystatus", + description="Find pets by status", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Create mock server with camelCase allowed_tools (as from OpenAPI spec) + server = MCPServer( + server_id="test-server", + name="per_store", + transport=MCPTransport.http, + allowed_tools=["addPet", "updatePet", "findPetsByStatus"], + ) + + # Filter tools + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return 3 tools (case-insensitive match) + assert len(filtered_tools) == 3 + assert any(t.name == "per_store-addpet" for t in filtered_tools) + assert any(t.name == "per_store-updatepet" for t in filtered_tools) + assert any(t.name == "per_store-findpetsbystatus" for t in filtered_tools) + assert not any(t.name == "per_store-deletepet" for t in filtered_tools) + + +def test_filter_tools_by_allowed_tools_no_filter(): + """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + tools = [ + MCPTool( + name="fusion_litellm_mcp-model_list", + description="List models", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="fusion_litellm_mcp-chat_completion", + description="Chat completion", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Server with no allowed_tools filter + server = MCPServer( + server_id="test-server", + name="fusion_litellm_mcp", + transport=MCPTransport.http, + allowed_tools=None, + ) + + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return all tools when no filter is configured + assert len(filtered_tools) == 2 diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 193b014f03..c43621d7f7 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -21,6 +21,140 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) +def test_get_team_models_all_proxy_models_includes_access_groups(): + """ + When a team has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names (e.g. 'claude-model-group') + in addition to individual model names. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=True + ) + assert "group-a" in result + assert "group-b" in result + assert "model1" in result + assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" + + +def test_get_team_models_all_proxy_models_without_include_flag(): + """ + When include_model_access_groups=False, access group names should NOT + appear in the result even with 'all-proxy-models'. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=False + ) + assert "group-a" not in result + assert "group-b" not in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_all_proxy_models_includes_access_groups(): + """ + When a key has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["all-proxy-models"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" + + +def test_get_key_models_passes_include_model_access_groups(): + """ + When a key explicitly has an access group name in its models list and + include_model_access_groups=True, the group name should be retained + (not stripped by _get_models_from_access_groups). + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["group-a"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1", "model2"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_does_not_mutate_input(): + """ + get_key_models must not mutate user_api_key_dict.models in-place. + _get_models_from_access_groups uses .pop()/.extend() which would corrupt + cached UserAPIKeyAuth objects if all_models were an alias instead of a copy. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + original_models = ["group-a", "extra-model"] + user_api_key_dict = UserAPIKeyAuth( + models=list(original_models), # give it a list + api_key="test-key", + ) + model_access_groups = { + "group-a": ["model1", "model2"], + } + + _ = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=["model1", "model2"], + model_access_groups=model_access_groups, + include_model_access_groups=False, + ) + # The original models list on the auth object must be unchanged + assert user_api_key_dict.models == original_models + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 5af24f9612..0b3a389409 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -2410,12 +2410,13 @@ def test_mapped_pass_through_routes_with_server_root_path(): ) + @pytest.mark.asyncio async def test_multipart_passthrough_preserves_boundary(): """ Test that multipart/form-data requests through passthrough preserve the boundary and can be correctly parsed by the upstream server. - + Regression test for multipart boundary stripping issue. """ from io import BytesIO @@ -2426,41 +2427,41 @@ async def test_multipart_passthrough_preserves_boundary(): mock_response.headers = httpx.Headers({"content-type": "application/json"}) mock_response.aread = AsyncMock(return_value=b'{"filename": "test.txt", "size": 17}') mock_response.text = '{"filename": "test.txt", "size": 17}' - + async def mock_httpx_request(method, url, **kwargs): # Verify that files parameter is passed (not json) assert "files" in kwargs, "Files should be passed for multipart requests" assert "file" in kwargs["files"], "File field should be in files dict" - + # Verify content-type is NOT in headers (httpx will set it with correct boundary) headers = kwargs.get("headers", {}) assert "content-type" not in headers, "content-type should be removed for multipart" - + filename, content, content_type = kwargs["files"]["file"] assert filename == "test.txt" assert content == b"test file content" assert content_type == "text/plain" - + return mock_response - + async_client = MagicMock() async_client.request = AsyncMock(side_effect=mock_httpx_request) - + # Create mock request request = MagicMock(spec=Request) request.method = "POST" request.headers = Headers({"content-type": "multipart/form-data; boundary=test123"}) - + # Mock form data file_content = b"test file content" file = BytesIO(file_content) headers = Headers({"content-type": "text/plain"}) upload_file = UploadFile(file=file, filename="test.txt", headers=headers) upload_file.read = AsyncMock(return_value=file_content) - + form_data = {"file": upload_file} request.form = AsyncMock(return_value=form_data) - + # Test the multipart handler directly response = await HttpPassThroughEndpointHelpers.make_multipart_http_request( request=request, @@ -2469,7 +2470,7 @@ async def test_multipart_passthrough_preserves_boundary(): headers={}, requested_query_params=None, ) - + # Verify the response assert response.status_code == 200 async_client.request.assert_called_once() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9a64e641b5..3249a7ec79 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1071,9 +1071,10 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), - # perform_redaction returns {"text": "redacted-by-litellm"} + # perform_redaction redacts content in-place within the choices structure parsed_response = json.loads(response_result) - assert parsed_response == {"text": "redacted-by-litellm"} + assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert parsed_response["choices"][0]["message"]["role"] == "assistant" @patch("litellm.secret_managers.main.get_secret_bool") diff --git a/tests/test_litellm/proxy/test_openapi_schema_validation.py b/tests/test_litellm/proxy/test_openapi_schema_validation.py new file mode 100644 index 0000000000..aafe08f303 --- /dev/null +++ b/tests/test_litellm/proxy/test_openapi_schema_validation.py @@ -0,0 +1,142 @@ +""" +Test that the OpenAPI schema generated by FastAPI is valid for specific endpoints. + +Validates fixes for: +- /spend/calculate response schema (must use proper OpenAPI 3.x content wrapper) +- /credentials/by_model/{model_id} path parameter (must not leak credential_name) + +Related issue: https://github.com/BerriAI/litellm/issues/21305 +""" + +import pytest + + +class TestSpendCalculateOpenAPISchema: + """Test /spend/calculate response schema is valid OpenAPI 3.x.""" + + def test_response_schema_has_description(self): + """The 200 response must have a 'description' field per OpenAPI 3.x spec.""" + from litellm.proxy.spend_tracking.spend_management_endpoints import router + + for route in router.routes: + if hasattr(route, "path") and route.path == "/spend/calculate": + responses = route.responses or {} + response_200 = responses.get(200, {}) + assert "description" in response_200, ( + "/spend/calculate 200 response must have a 'description' field" + ) + break + else: + pytest.fail("/spend/calculate route not found in router") + + def test_response_schema_has_content_wrapper(self): + """The 200 response must use 'content' wrapper, not bare properties.""" + from litellm.proxy.spend_tracking.spend_management_endpoints import router + + for route in router.routes: + if hasattr(route, "path") and route.path == "/spend/calculate": + responses = route.responses or {} + response_200 = responses.get(200, {}) + # Must NOT have 'cost' as a top-level key (invalid OpenAPI) + assert "cost" not in response_200, ( + "/spend/calculate 200 response must not have 'cost' as a " + "top-level property - use 'content' wrapper instead" + ) + # Must have 'content' wrapper + assert "content" in response_200, ( + "/spend/calculate 200 response must have a 'content' field" + ) + content = response_200["content"] + assert "application/json" in content + assert "schema" in content["application/json"] + break + else: + pytest.fail("/spend/calculate route not found in router") + + +class TestCredentialEndpointsOpenAPISchema: + """Test /credentials endpoints have correct path parameters.""" + + def test_by_name_and_by_model_are_separate_handlers(self): + """ + /credentials/by_name/{credential_name} and /credentials/by_model/{model_id} + must be separate handler functions so each only declares its own path params. + """ + from litellm.proxy.credential_endpoints.endpoints import router + + by_name_routes = [] + by_model_routes = [] + for route in router.routes: + if not hasattr(route, "path"): + continue + if "by_name" in route.path: + by_name_routes.append(route) + elif "by_model" in route.path: + by_model_routes.append(route) + + assert len(by_name_routes) == 1, "Expected exactly one by_name route" + assert len(by_model_routes) == 1, "Expected exactly one by_model route" + + # They must be different endpoint functions + by_name_endpoint = by_name_routes[0].endpoint + by_model_endpoint = by_model_routes[0].endpoint + assert by_name_endpoint is not by_model_endpoint, ( + "by_name and by_model must be separate handler functions " + "to avoid path parameter conflicts in OpenAPI spec" + ) + + def test_by_model_route_does_not_require_credential_name(self): + """ + The /credentials/by_model/{model_id} route must NOT have + credential_name as a parameter. + """ + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_model, + ) + + sig = inspect.signature(get_credential_by_model) + param_names = list(sig.parameters.keys()) + assert "credential_name" not in param_names, ( + "get_credential_by_model must not have a credential_name parameter" + ) + + def test_by_name_route_does_not_require_model_id(self): + """ + The /credentials/by_name/{credential_name} route must NOT have + model_id as a parameter. + """ + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_name, + ) + + sig = inspect.signature(get_credential_by_name) + param_names = list(sig.parameters.keys()) + assert "model_id" not in param_names, ( + "get_credential_by_name must not have a model_id parameter" + ) + + def test_by_model_has_model_id_path_param(self): + """The by_model handler must accept model_id as a path parameter.""" + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_model, + ) + + sig = inspect.signature(get_credential_by_model) + assert "model_id" in sig.parameters, ( + "get_credential_by_model must have a model_id parameter" + ) + + def test_by_name_has_credential_name_path_param(self): + """The by_name handler must accept credential_name as a path parameter.""" + import inspect + from litellm.proxy.credential_endpoints.endpoints import ( + get_credential_by_name, + ) + + sig = inspect.signature(get_credential_by_name) + assert "credential_name" in sig.parameters, ( + "get_credential_by_name must have a credential_name parameter" + ) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 6d6162437c..a931a9bc93 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -1774,3 +1774,128 @@ class TestStreamingIDConsistency: # Verify it matches the cached ID assert iterator._cached_item_id is not None assert iterator._cached_item_id == text_done_id + + def test_parallel_tool_calls_merged_into_single_assistant_message(self): + """ + Regression test: multi-turn parallel tool calls via the Responses API must + produce a single assistant message with all tool_calls, not one assistant + message per function_call item. + + When the model responds with two parallel tool calls (e.g. get_weather for + SF and NYC), the next Responses API request includes two consecutive + function_call items followed by two function_call_output items. + + Without the fix each function_call becomes its own assistant message, + producing back-to-back assistant messages that Anthropic/Vertex AI rejects: + "tool_use ids were found without tool_result blocks immediately after". + """ + input_items = [ + {"type": "message", "role": "user", "content": "Weather in SF and NYC?"}, + # Two parallel tool calls from the previous assistant response + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": '{"city": "SF"}', + }, + { + "type": "function_call", + "call_id": "toolu_02", + "name": "get_weather", + "arguments": '{"city": "NYC"}', + }, + # Tool results + {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, + {"type": "function_call_output", "call_id": "toolu_02", "output": "55°F"}, + ] + + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=input_items + ) + + roles = [ + m.get("role") if isinstance(m, dict) else getattr(m, "role", None) + for m in messages + ] + + # Must not have two consecutive assistant messages + for i in range(len(roles) - 1): + assert not ( + roles[i] == "assistant" and roles[i + 1] == "assistant" + ), f"Consecutive assistant messages at indices {i} and {i+1}: {roles}" + + # The single assistant message must contain BOTH tool_calls + assistant_messages = [ + m for m in messages + if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) + == "assistant" + ] + assert len(assistant_messages) == 1, ( + f"Expected 1 assistant message, got {len(assistant_messages)}" + ) + + assistant_msg = assistant_messages[0] + tool_calls = ( + assistant_msg.get("tool_calls") + if isinstance(assistant_msg, dict) + else getattr(assistant_msg, "tool_calls", None) + ) + assert tool_calls is not None and len(tool_calls) == 2, ( + f"Expected 2 tool_calls in the merged assistant message, got: {tool_calls}" + ) + + call_ids = [ + (tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)) + for tc in tool_calls + ] + assert "toolu_01" in call_ids, f"toolu_01 missing from tool_calls: {call_ids}" + assert "toolu_02" in call_ids, f"toolu_02 missing from tool_calls: {call_ids}" + + # Both tool messages must be present + tool_messages = [ + m for m in messages + if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) + == "tool" + ] + assert len(tool_messages) == 2, ( + f"Expected 2 tool messages, got {len(tool_messages)}" + ) + + def test_single_tool_call_still_works_after_merge_fix(self): + """ + Ensure the parallel-tool-call merging fix does not break the existing + single-tool-call path. + """ + input_items = [ + {"type": "message", "role": "user", "content": "Weather in SF?"}, + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": '{"city": "SF"}', + }, + {"type": "function_call_output", "call_id": "toolu_01", "output": "72°F"}, + ] + + messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( + input=input_items + ) + + roles = [ + m.get("role") if isinstance(m, dict) else getattr(m, "role", None) + for m in messages + ] + + assert "user" in roles + assert "assistant" in roles + assert "tool" in roles + + assistant_messages = [m for m in messages if (m.get("role") if isinstance(m, dict) else getattr(m, "role", None)) == "assistant"] + assert len(assistant_messages) == 1 + + tool_calls = ( + assistant_messages[0].get("tool_calls") + if isinstance(assistant_messages[0], dict) + else getattr(assistant_messages[0], "tool_calls", None) + ) + assert tool_calls is not None and len(tool_calls) == 1 diff --git a/tests/test_litellm/test_router_retry_non_retryable_errors.py b/tests/test_litellm/test_router_retry_non_retryable_errors.py new file mode 100644 index 0000000000..20a1c979a0 --- /dev/null +++ b/tests/test_litellm/test_router_retry_non_retryable_errors.py @@ -0,0 +1,251 @@ +""" +Test that the Router retry loop correctly handles non-retryable errors. + +Verifies that: +1. Non-retryable errors (e.g., 400 ContextWindowExceeded) inside the retry loop + break out immediately instead of being swallowed. +2. original_exception is updated to the latest error, not stuck on the first. +3. Retryable errors (e.g., 429 RateLimitError) still retry normally. + +Regression tests for https://github.com/BerriAI/litellm/issues/21343 +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm import Router + + +def _make_rate_limit_error(message="Rate limited"): + """Create a RateLimitError for testing.""" + return litellm.RateLimitError( + message=message, + llm_provider="bedrock", + model="anthropic.claude-v2", + ) + + +def _make_context_window_error(message="prompt is too long: 1205821 tokens > 200000"): + """Create a ContextWindowExceededError for testing.""" + return litellm.ContextWindowExceededError( + message=message, + llm_provider="vertex_ai", + model="claude-3-opus", + ) + + +def _make_bad_request_error(message="Invalid request"): + """Create a BadRequestError for testing.""" + return litellm.BadRequestError( + message=message, + llm_provider="openai", + model="gpt-4", + ) + + +def _make_not_found_error(message="Model not found"): + """Create a NotFoundError for testing.""" + return litellm.NotFoundError( + message=message, + llm_provider="openai", + model="gpt-99", + ) + + +def _create_router(num_retries=2): + """Create a Router with two deployments for testing.""" + return Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "fake-key-1", + }, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "openai/gpt-4", + "api_key": "fake-key-2", + }, + }, + ], + num_retries=num_retries, + ) + + +def _base_kwargs(): + """Return kwargs required by async_function_with_retries.""" + return { + "model": "test-model", + "messages": [{"role": "user", "content": "test"}], + "original_function": AsyncMock(), + "metadata": {}, + } + + +@pytest.mark.asyncio +async def test_non_retryable_error_in_retry_loop_raises_immediately(): + """ + When a non-retryable error (400 ContextWindowExceeded) occurs inside the + retry loop, the router should raise it immediately instead of swallowing it + and raising the original error. + + Scenario: First call -> 429, Retry -> 400 (non-retryable) + Expected: ContextWindowExceededError is raised, NOT RateLimitError + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + context_window_error = _make_context_window_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise context_window_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.ContextWindowExceededError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + +@pytest.mark.asyncio +async def test_bad_request_error_in_retry_loop_raises_immediately(): + """ + A generic 400 BadRequestError inside the retry loop should also break out + immediately since 400 is not retryable. + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + bad_request_error = _make_bad_request_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise bad_request_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.BadRequestError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + +@pytest.mark.asyncio +async def test_original_exception_updated_to_latest_error(): + """ + When all retries are exhausted with retryable errors, the LAST error + should be raised, not the first one. + """ + router = _create_router(num_retries=2) + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + raise _make_rate_limit_error(f"Rate limit attempt {call_count}") + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.RateLimitError) as exc_info: + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + # Should be the LAST error, not the first + assert "Rate limit attempt 3" in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_retryable_errors_still_retry_normally(): + """ + Retryable errors (429 RateLimitError) should still be retried the + configured number of times before raising. + """ + router = _create_router(num_retries=3) + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + raise _make_rate_limit_error(f"Rate limit attempt {call_count}") + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.RateLimitError): + await router.async_function_with_retries( + num_retries=3, + **_base_kwargs(), + ) + + # Initial call + 3 retries = 4 total calls + assert call_count == 4 + + +@pytest.mark.asyncio +async def test_not_found_error_in_retry_loop_raises_immediately(): + """ + A 404 NotFoundError inside the retry loop should break out immediately. + """ + router = _create_router(num_retries=2) + + rate_limit_error = _make_rate_limit_error() + not_found_error = _make_not_found_error() + + call_count = 0 + + async def mock_make_call(*args, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + raise rate_limit_error + else: + raise not_found_error + + with patch.object(router, "make_call", side_effect=mock_make_call), \ + patch.object(router, "_async_get_healthy_deployments", + return_value=(["d1", "d2"], ["d1", "d2"])), \ + patch.object(router, "_time_to_sleep_before_retry", return_value=0), \ + patch.object(router, "log_retry", side_effect=lambda kwargs, e: kwargs): + with pytest.raises(litellm.NotFoundError): + await router.async_function_with_retries( + num_retries=2, + **_base_kwargs(), + ) + + # Only 2 calls: initial + first retry that hits non-retryable + assert call_count == 2 diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 7d7bb924ac..418bca64ea 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -275,8 +275,8 @@ it("should display user email correctly", async () => { }); }); -it("should show skeleton loaders when isLoading is true", () => { - // Mock loading state +it("should show loading message only on initial load (isPending)", () => { + // Mock initial loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -296,7 +296,7 @@ it("should show skeleton loaders when isLoading is true", () => { renderWithProviders(); - // Check that loading message is shown + // Check that loading message is shown on initial load expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -810,3 +810,79 @@ describe("pagination display – total count and page count", () => { }); }); }); + +describe("refetch button", () => { + it("should show Fetch button in normal state", () => { + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeInTheDocument(); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); + + it("should show Fetching state and keep table data visible during refetch", () => { + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + // Button should show "Fetching" and be disabled + expect(screen.getByText("Fetching")).toBeInTheDocument(); + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeDisabled(); + + // Table data should still be visible (stale data) + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + + // "Loading keys..." should NOT appear during refetch + expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); + }); + + it("should call refetch when Fetch button is clicked", () => { + const mockRefetch = vi.fn(); + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: false, + refetch: mockRefetch, + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + fireEvent.click(fetchButton); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); + + it("should show Fetch button enabled on error so user can retry", () => { + mockUseKeys.mockReturnValue({ + data: null, + isPending: false, + isFetching: false, + isError: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index fd0cd4dd50..20cc1b8153 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -85,6 +85,7 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo data: keys, isPending: isLoading, isFetching, + isError, refetch, } = useKeys(tablePagination.pageIndex + 1, tablePagination.pageSize, { sortBy: sortBy || undefined, @@ -102,6 +103,15 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); + // Defer the transition so the button stays in loading state until the table + // has rendered with the new data (mirrors the spend-logs pattern) + const isFetchingDeferred = useDeferredValue(isFetching); + const isButtonLoading = (isFetching || isFetchingDeferred) && !isError; + + const handleRefresh = () => { + refetch(); + }; + const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -669,16 +679,28 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading || isFetching ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} +
+ {isLoading ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )} + + } + onClick={handleRefresh} + disabled={isButtonLoading} + title="Fetch data" + > + {isButtonLoading ? "Fetching" : "Fetch"} + +
- {isLoading || isFetching ? ( + {isLoading ? ( ) : ( @@ -686,24 +708,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : (