From 2c738cc939c408cd0e85772bd503297c7d363197 Mon Sep 17 00:00:00 2001 From: Maxwell Calkin <101308415+MaxwellCalkin@users.noreply.github.com> Date: Mon, 9 Mar 2026 22:51:25 -0400 Subject: [PATCH 01/20] fix: strip empty text content blocks in /v1/messages endpoint (#23097) Claude's API returns assistant messages with empty text blocks ({"type": "text", "text": ""}) alongside tool_use blocks during multi-turn tool-use conversations. These blocks are rejected when sent back to the API with "text content blocks must be non-empty". Sanitization already exists for other code paths (/v1/chat/completions for both Anthropic and Bedrock), but NOT for the /v1/messages native path. This adds the same treatment by stripping empty text blocks from messages in async_anthropic_messages_handler before they are forwarded to the provider. Fixes #22930 --- litellm/llms/custom_httpx/llm_http_handler.py | 60 +++++ ...est_v1_messages_empty_text_sanitization.py | 247 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1cef3e9ce1..6a5d669cad 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,6 +152,59 @@ else: LiteLLMLoggingObj = Any +def _sanitize_anthropic_messages_empty_text_blocks( + messages: List[Dict], +) -> List[Dict]: + """ + Strip empty text content blocks from Anthropic-format messages. + + Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` + alongside ``tool_use`` blocks, but rejects them when sent back in subsequent + requests. This helper removes those empty text blocks so the /v1/messages + native path doesn't forward them as-is. + + - If a content list contains a mix of empty text blocks and other blocks + (e.g. tool_use), the empty text blocks are removed. + - If *all* blocks in a content list are empty text, the content is replaced + with a single non-empty placeholder to avoid sending an empty array. + + Ref: https://github.com/BerriAI/litellm/issues/22930 + """ + sanitized: List[Dict] = [] + for message in messages: + content = message.get("content") + if not isinstance(content, list): + sanitized.append(message) + continue + + filtered = [ + block + for block in content + if not ( + isinstance(block, dict) + and block.get("type") == "text" + and not block.get("text", "").strip() + ) + ] + + if filtered == content: + # Nothing was removed — keep original message as-is. + sanitized.append(message) + elif filtered: + # Some empty text blocks removed, but other content remains. + new_message = message.copy() + new_message["content"] = filtered + sanitized.append(new_message) + else: + # All blocks were empty text blocks. Replace with a placeholder + # so we don't send an empty content array. + new_message = message.copy() + new_message["content"] = [{"type": "text", "text": "..."}] + sanitized.append(new_message) + + return sanitized + + class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1905,6 +1958,13 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) + # Sanitize empty text content blocks from messages before forwarding. + # Claude's API returns assistant messages with empty text blocks + # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects + # them when sent back. Strip these to prevent 400 errors. + # Ref: https://github.com/BerriAI/litellm/issues/22930 + messages = _sanitize_anthropic_messages_empty_text_blocks(messages) + # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py new file mode 100644 index 0000000000..b397b5a484 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py @@ -0,0 +1,247 @@ +""" +Test empty text content block sanitization for the /v1/messages native path. + +The Anthropic API returns assistant messages with empty text blocks +({"type": "text", "text": ""}) alongside tool_use blocks, but rejects +them when sent back. The /v1/messages endpoint must strip these before +forwarding to providers. + +Ref: https://github.com/BerriAI/litellm/issues/22930 +""" + +import pytest + +from litellm.llms.custom_httpx.llm_http_handler import ( + _sanitize_anthropic_messages_empty_text_blocks, +) + + +class TestSanitizeAnthropicMessagesEmptyTextBlocks: + """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" + + def test_strips_empty_text_alongside_tool_use(self): + """ + The most common case from the bug report: an assistant message + containing an empty text block next to a tool_use block. + """ + messages = [ + {"role": "user", "content": "Run the command."}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_xxx", + "name": "Bash", + "input": {"command": "ls"}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result) == 2 + assert result[0] == messages[0] # user message unchanged + # assistant content should only have the tool_use block + assert len(result[1]["content"]) == 1 + assert result[1]["content"][0]["type"] == "tool_use" + + def test_preserves_nonempty_text_blocks(self): + """Non-empty text blocks must not be removed.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me check that."}, + { + "type": "tool_use", + "id": "toolu_yyy", + "name": "Bash", + "input": {"command": "pwd"}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} + + def test_whitespace_only_text_block_stripped(self): + """Whitespace-only text blocks should also be stripped.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": " \n\t "}, + { + "type": "tool_use", + "id": "toolu_zzz", + "name": "Bash", + "input": {}, + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["type"] == "tool_use" + + def test_all_empty_text_blocks_replaced_with_placeholder(self): + """ + If all content blocks are empty text, replace with a placeholder + to avoid sending an empty content array. + """ + messages = [ + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][0]["text"].strip() # must be non-empty + + def test_string_content_untouched(self): + """Messages with string content should pass through unchanged.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_no_content_key_untouched(self): + """Messages without a content key should pass through.""" + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant"}, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_user_message_content_list_also_sanitized(self): + """ + Empty text blocks should be stripped from user messages too, + not just assistant messages. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": "actual question"}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert len(result[0]["content"]) == 1 + assert result[0]["content"][0]["text"] == "actual question" + + def test_tool_result_content_blocks_untouched(self): + """ + tool_result content blocks should not be affected — only + {"type": "text", "text": ""} blocks are stripped. + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_xxx", + "content": "", + }, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + assert result == messages + + def test_multiple_messages_mixed(self): + """End-to-end scenario with multiple messages, some needing sanitization.""" + messages = [ + {"role": "user", "content": "Run ls"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_1", + "name": "Bash", + "input": {"command": "ls"}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": "file1.txt\nfile2.txt", + }, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here are the files:"}, + ], + }, + ] + + result = _sanitize_anthropic_messages_empty_text_blocks(messages) + + # First message: string content, unchanged + assert result[0] == messages[0] + # Second message: empty text stripped, only tool_use remains + assert len(result[1]["content"]) == 1 + assert result[1]["content"][0]["type"] == "tool_use" + # Third message: tool_result, unchanged + assert result[2] == messages[2] + # Fourth message: non-empty text, unchanged + assert result[3] == messages[3] + + def test_does_not_mutate_original_messages(self): + """The function should not modify the input list or its dicts.""" + original_content = [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "toolu_1", + "name": "Bash", + "input": {}, + }, + ] + messages = [ + { + "role": "assistant", + "content": original_content, + }, + ] + + _sanitize_anthropic_messages_empty_text_blocks(messages) + + # Original message content should be unchanged + assert len(messages[0]["content"]) == 2 + assert messages[0]["content"][0] == {"type": "text", "text": ""} From 30b82c3a0cef9ffa3abcdbd1768ee9cc026f3825 Mon Sep 17 00:00:00 2001 From: tristanolive Date: Tue, 10 Mar 2026 03:46:43 +0000 Subject: [PATCH 02/20] feat(charity_engine): add Charity Engine provider (#23223) * feat(charity_engine): add Charity Engine provider Charity Engine is a crowdsourced distributed computing platform that donates processing power to charitable causes. Its inference API provides OpenAI-compatible chat, completions, and embeddings endpoints. * test(charity_engine): add provider config and resolution tests Verify JSONProviderRegistry config, provider list membership, model routing for charity_engine/, and Router compatibility. * feat(charity_engine): add Charity Engine to LlmProviders enum Enables provider_list membership and LlmProviders.CHARITY_ENGINE resolution required by the provider and test suite. * fix(charity_engine): remove api_base_env to fix non-deterministic test The CHARITY_ENGINE_API_BASE env var could override the base_url in CI, causing test_charity_engine_provider_resolution to fail intermittently. * fix(charity_engine): remove trailing slash from base_url --- litellm/llms/openai_like/providers.json | 7 ++ litellm/types/utils.py | 1 + .../llms/openai_like/test_charity_engine.py | 101 ++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/test_litellm/llms/openai_like/test_charity_engine.py diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b3125d4ad3..275c352b39 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -94,5 +94,12 @@ "assemblyai": { "base_url": "https://llm-gateway.assemblyai.com/v1", "api_key_env": "ASSEMBLYAI_API_KEY" + }, + "charity_engine": { + "base_url": "https://api.charityengine.services/remotejobs/v2/inference", + "api_key_env": "CHARITY_ENGINE_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8ae0cf2892..b5d5c06924 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3177,6 +3177,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" SAP_GENERATIVE_AI_HUB = "sap" ASSEMBLYAI = "assemblyai" + CHARITY_ENGINE = "charity_engine" GITHUB_COPILOT = "github_copilot" SNOWFLAKE = "snowflake" GRADIENT_AI = "gradient_ai" diff --git a/tests/test_litellm/llms/openai_like/test_charity_engine.py b/tests/test_litellm/llms/openai_like/test_charity_engine.py new file mode 100644 index 0000000000..5d6a751b62 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_charity_engine.py @@ -0,0 +1,101 @@ +""" +Tests for Charity Engine provider configuration and integration. +""" + +import os +import sys + +try: + import pytest +except ImportError: + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + +import litellm + + +class TestCharityEngineProviderConfig: + """Test Charity Engine provider configuration""" + + def test_charity_engine_in_provider_list(self): + """Test that charity_engine is in the provider list""" + from litellm import LlmProviders + + assert hasattr(LlmProviders, "CHARITY_ENGINE") + assert LlmProviders.CHARITY_ENGINE.value == "charity_engine" + assert "charity_engine" in litellm.provider_list + + def test_charity_engine_json_config_exists(self): + """Test that charity_engine is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("charity_engine") + + charity_engine = JSONProviderRegistry.get("charity_engine") + assert charity_engine is not None + assert charity_engine.base_url == "https://api.charityengine.services/remotejobs/v2/inference" + assert charity_engine.api_key_env == "CHARITY_ENGINE_API_KEY" + assert charity_engine.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_charity_engine_provider_resolution(self): + """Test that provider resolution finds charity_engine""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="charity_engine/gemma3:270m", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "gemma3:270m" + assert provider == "charity_engine" + assert api_base == "https://api.charityengine.services/remotejobs/v2/inference" + + def test_charity_engine_router_config(self): + """Test that charity_engine can be used in Router configuration""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "gemma3-270m", + "litellm_params": { + "model": "charity_engine/gemma3:270m", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "gemma3-270m" + + +if __name__ == "__main__": + print("Testing Charity Engine Provider...") + + test_config = TestCharityEngineProviderConfig() + + print("\n1. Testing provider in list...") + test_config.test_charity_engine_in_provider_list() + print(" ✓ charity_engine in provider list") + + print("\n2. Testing JSON config...") + test_config.test_charity_engine_json_config_exists() + print(" ✓ charity_engine JSON config loaded") + + print("\n3. Testing provider resolution...") + test_config.test_charity_engine_provider_resolution() + print(" ✓ Provider resolution works") + + print("\n4. Testing router configuration...") + test_config.test_charity_engine_router_config() + print(" ✓ Router configuration works") + + print("\n" + "=" * 50) + print("✓ All configuration tests passed!") + print("=" * 50) From dd6f0d6c55179634b5a5b8d1f670d97c56891b6c Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 9 Mar 2026 20:56:27 -0700 Subject: [PATCH 03/20] fix: forward recognized OpenAI params from kwargs in completion() (#23224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that arrives via completion(**kwargs) is now automatically forwarded to get_optional_params(), even if it's not a named parameter of completion(). Previously, get_non_default_completion_params() excluded params in OPENAI_CHAT_COMPLETION_PARAMS (assuming they'd be forwarded via the named-param path), while optional_param_args only contained explicitly named params. Params like 'store' that were in the known-params list but not named params fell through both paths and were silently dropped. The fix adds a 7-line loop after building optional_param_args that forwards any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES. This means new OpenAI params only need to be added to the constants dict — no boilerplate changes to 3+ function signatures required. Fixes #23087 Co-authored-by: Cursor Agent --- litellm/main.py | 8 + .../llms/openai/chat/test_store_param.py | 188 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index 364519e1fe..e23baadb79 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( + DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1487,6 +1488,13 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py new file mode 100644 index 0000000000..0fd4799dae --- /dev/null +++ b/tests/test_litellm/llms/openai/chat/test_store_param.py @@ -0,0 +1,188 @@ +""" +Tests for the `store` parameter being correctly forwarded to OpenAI. + +Related issue: https://github.com/BerriAI/litellm/issues/23087 + +The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and +DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because +get_non_default_completion_params() excluded it (as a "known" param) +while optional_param_args didn't include it (not a named param of +completion()). The fix adds a safety net in completion() that forwards +any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't +already in optional_param_args. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES +from litellm.utils import get_non_default_completion_params, get_optional_params + + +class TestStoreParamForwarding: + """Tests that `store` flows through the parameter processing pipeline.""" + + def test_store_true_forwarded_for_openai(self): + """should forward store=True for OpenAI models via kwargs""" + result = get_optional_params( + model="gpt-5.1", + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True + + def test_store_false_forwarded_for_openai(self): + """should forward store=False for OpenAI models""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + store=False, + ) + assert result.get("store") is False + + def test_store_none_not_forwarded(self): + """should not include store when it is None (default)""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "store" not in result + + def test_store_with_gpt5_models(self): + """should forward store=True for GPT-5 family models""" + for model in ["gpt-5.1", "gpt-5.2"]: + result = get_optional_params( + model=model, + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True, f"store not forwarded for {model}" + + def test_store_in_supported_params(self): + """should list store as a supported OpenAI param""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model) + assert "store" in supported, f"store not in supported params for {model}" + + def test_store_in_transform_request(self): + """should include store in the final transformed request body""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + + def test_store_true_with_metadata(self): + """should forward both store and metadata when both are set""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True, "metadata": {"key": "value"}} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + assert result.get("metadata") == {"key": "value"} + + +class TestDefaultParamValuesSafetyNet: + """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows + through completion() even without being a named parameter.""" + + def test_known_openai_param_excluded_from_non_default(self): + """should confirm get_non_default_completion_params excludes known OpenAI params""" + kwargs = {"store": True, "temperature": 0.5} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert "store" not in non_default + assert "temperature" not in non_default + + def test_unknown_param_included_in_non_default(self): + """should pass through unknown provider-specific params""" + kwargs = {"my_custom_provider_param": "foo"} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert non_default.get("my_custom_provider_param") == "foo" + + def test_safety_net_forwards_recognized_kwargs(self): + """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + that are not already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"store": True, "metadata": {"key": "value"}} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["store"] is True + assert optional_param_args["metadata"] == {"key": "value"} + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_does_not_override_existing(self): + """should not override a param that's already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"temperature": 0.9} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_skips_none_values(self): + """should not forward params with None value (the default)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": None} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert "store" not in optional_param_args + + def test_safety_net_forwards_falsy_non_none(self): + """should forward store=False (falsy but not None)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": False} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args.get("store") is False From 325df8d62aaaaa8d079ed2269b781dcdfcd0202a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:45:28 +0530 Subject: [PATCH 04/20] Fix logging tests --- litellm/litellm_core_utils/redact_messages.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ad68f3851a..ddeb24d04a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -123,6 +123,16 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] + elif isinstance(_result, dict) and "choices" in _result: + # ModelResponse.model_dump() returns dict - redact choices in place + if isinstance(_result.get("choices"), list) and len(_result["choices"]) > 0: + choice = _result["choices"][0] + if isinstance(choice, dict) and "message" in choice: + msg = choice["message"] + if isinstance(msg, dict) and "content" in msg: + msg["content"] = "redacted-by-litellm" + if isinstance(msg, dict) and "audio" in msg: + msg["audio"] = None else: return {"text": "redacted-by-litellm"} return _result From 56be0a651f2d355be8ea9a0a1f2f71b2dc56cf80 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:50:37 +0530 Subject: [PATCH 05/20] fix: add charity_engine to provider_endpoints_support.json Made-with: Cursor --- provider_endpoints_support.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { From c1b860b3c1f98c94892eaf30ddf7b32a46e20194 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:53:19 +0530 Subject: [PATCH 06/20] Revert "fix: strip empty text content blocks in /v1/messages endpoint (#23097)" This reverts commit 2c738cc939c408cd0e85772bd503297c7d363197. --- litellm/llms/custom_httpx/llm_http_handler.py | 60 ----- ...est_v1_messages_empty_text_sanitization.py | 247 ------------------ 2 files changed, 307 deletions(-) delete mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6a5d669cad..1cef3e9ce1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,59 +152,6 @@ else: LiteLLMLoggingObj = Any -def _sanitize_anthropic_messages_empty_text_blocks( - messages: List[Dict], -) -> List[Dict]: - """ - Strip empty text content blocks from Anthropic-format messages. - - Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` - alongside ``tool_use`` blocks, but rejects them when sent back in subsequent - requests. This helper removes those empty text blocks so the /v1/messages - native path doesn't forward them as-is. - - - If a content list contains a mix of empty text blocks and other blocks - (e.g. tool_use), the empty text blocks are removed. - - If *all* blocks in a content list are empty text, the content is replaced - with a single non-empty placeholder to avoid sending an empty array. - - Ref: https://github.com/BerriAI/litellm/issues/22930 - """ - sanitized: List[Dict] = [] - for message in messages: - content = message.get("content") - if not isinstance(content, list): - sanitized.append(message) - continue - - filtered = [ - block - for block in content - if not ( - isinstance(block, dict) - and block.get("type") == "text" - and not block.get("text", "").strip() - ) - ] - - if filtered == content: - # Nothing was removed — keep original message as-is. - sanitized.append(message) - elif filtered: - # Some empty text blocks removed, but other content remains. - new_message = message.copy() - new_message["content"] = filtered - sanitized.append(new_message) - else: - # All blocks were empty text blocks. Replace with a placeholder - # so we don't send an empty content array. - new_message = message.copy() - new_message["content"] = [{"type": "text", "text": "..."}] - sanitized.append(new_message) - - return sanitized - - class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1958,13 +1905,6 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) - # Sanitize empty text content blocks from messages before forwarding. - # Claude's API returns assistant messages with empty text blocks - # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects - # them when sent back. Strip these to prevent 400 errors. - # Ref: https://github.com/BerriAI/litellm/issues/22930 - messages = _sanitize_anthropic_messages_empty_text_blocks(messages) - # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py deleted file mode 100644 index b397b5a484..0000000000 --- a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Test empty text content block sanitization for the /v1/messages native path. - -The Anthropic API returns assistant messages with empty text blocks -({"type": "text", "text": ""}) alongside tool_use blocks, but rejects -them when sent back. The /v1/messages endpoint must strip these before -forwarding to providers. - -Ref: https://github.com/BerriAI/litellm/issues/22930 -""" - -import pytest - -from litellm.llms.custom_httpx.llm_http_handler import ( - _sanitize_anthropic_messages_empty_text_blocks, -) - - -class TestSanitizeAnthropicMessagesEmptyTextBlocks: - """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" - - def test_strips_empty_text_alongside_tool_use(self): - """ - The most common case from the bug report: an assistant message - containing an empty text block next to a tool_use block. - """ - messages = [ - {"role": "user", "content": "Run the command."}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_xxx", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result) == 2 - assert result[0] == messages[0] # user message unchanged - # assistant content should only have the tool_use block - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - - def test_preserves_nonempty_text_blocks(self): - """Non-empty text blocks must not be removed.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me check that."}, - { - "type": "tool_use", - "id": "toolu_yyy", - "name": "Bash", - "input": {"command": "pwd"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 2 - assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} - - def test_whitespace_only_text_block_stripped(self): - """Whitespace-only text blocks should also be stripped.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": " \n\t "}, - { - "type": "tool_use", - "id": "toolu_zzz", - "name": "Bash", - "input": {}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "tool_use" - - def test_all_empty_text_blocks_replaced_with_placeholder(self): - """ - If all content blocks are empty text, replace with a placeholder - to avoid sending an empty content array. - """ - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "text" - assert result[0]["content"][0]["text"].strip() # must be non-empty - - def test_string_content_untouched(self): - """Messages with string content should pass through unchanged.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_no_content_key_untouched(self): - """Messages without a content key should pass through.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_user_message_content_list_also_sanitized(self): - """ - Empty text blocks should be stripped from user messages too, - not just assistant messages. - """ - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "actual question"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["text"] == "actual question" - - def test_tool_result_content_blocks_untouched(self): - """ - tool_result content blocks should not be affected — only - {"type": "text", "text": ""} blocks are stripped. - """ - messages = [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_xxx", - "content": "", - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_multiple_messages_mixed(self): - """End-to-end scenario with multiple messages, some needing sanitization.""" - messages = [ - {"role": "user", "content": "Run ls"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "file1.txt\nfile2.txt", - }, - ], - }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Here are the files:"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - # First message: string content, unchanged - assert result[0] == messages[0] - # Second message: empty text stripped, only tool_use remains - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - # Third message: tool_result, unchanged - assert result[2] == messages[2] - # Fourth message: non-empty text, unchanged - assert result[3] == messages[3] - - def test_does_not_mutate_original_messages(self): - """The function should not modify the input list or its dicts.""" - original_content = [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {}, - }, - ] - messages = [ - { - "role": "assistant", - "content": original_content, - }, - ] - - _sanitize_anthropic_messages_empty_text_blocks(messages) - - # Original message content should be unchanged - assert len(messages[0]["content"]) == 2 - assert messages[0]["content"][0] == {"type": "text", "text": ""} From 2cb47727b62d63417d41032aeefb7728df1a3442 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 9 Mar 2026 20:56:27 -0700 Subject: [PATCH 07/20] fix: forward recognized OpenAI params from kwargs in completion() (#23224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that arrives via completion(**kwargs) is now automatically forwarded to get_optional_params(), even if it's not a named parameter of completion(). Previously, get_non_default_completion_params() excluded params in OPENAI_CHAT_COMPLETION_PARAMS (assuming they'd be forwarded via the named-param path), while optional_param_args only contained explicitly named params. Params like 'store' that were in the known-params list but not named params fell through both paths and were silently dropped. The fix adds a 7-line loop after building optional_param_args that forwards any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES. This means new OpenAI params only need to be added to the constants dict — no boilerplate changes to 3+ function signatures required. Fixes #23087 Co-authored-by: Cursor Agent --- litellm/main.py | 8 + .../llms/openai/chat/test_store_param.py | 188 ++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index 364519e1fe..e23baadb79 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( + DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1487,6 +1488,13 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py new file mode 100644 index 0000000000..0fd4799dae --- /dev/null +++ b/tests/test_litellm/llms/openai/chat/test_store_param.py @@ -0,0 +1,188 @@ +""" +Tests for the `store` parameter being correctly forwarded to OpenAI. + +Related issue: https://github.com/BerriAI/litellm/issues/23087 + +The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and +DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because +get_non_default_completion_params() excluded it (as a "known" param) +while optional_param_args didn't include it (not a named param of +completion()). The fix adds a safety net in completion() that forwards +any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't +already in optional_param_args. +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES +from litellm.utils import get_non_default_completion_params, get_optional_params + + +class TestStoreParamForwarding: + """Tests that `store` flows through the parameter processing pipeline.""" + + def test_store_true_forwarded_for_openai(self): + """should forward store=True for OpenAI models via kwargs""" + result = get_optional_params( + model="gpt-5.1", + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True + + def test_store_false_forwarded_for_openai(self): + """should forward store=False for OpenAI models""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + store=False, + ) + assert result.get("store") is False + + def test_store_none_not_forwarded(self): + """should not include store when it is None (default)""" + result = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + ) + assert "store" not in result + + def test_store_with_gpt5_models(self): + """should forward store=True for GPT-5 family models""" + for model in ["gpt-5.1", "gpt-5.2"]: + result = get_optional_params( + model=model, + custom_llm_provider="openai", + store=True, + ) + assert result.get("store") is True, f"store not forwarded for {model}" + + def test_store_in_supported_params(self): + """should list store as a supported OpenAI param""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: + supported = config.get_supported_openai_params(model) + assert "store" in supported, f"store not in supported params for {model}" + + def test_store_in_transform_request(self): + """should include store in the final transformed request body""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + + def test_store_true_with_metadata(self): + """should forward both store and metadata when both are set""" + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + + config = OpenAIGPTConfig() + messages = [{"role": "user", "content": "Hello"}] + optional_params = {"store": True, "metadata": {"key": "value"}} + result = config.transform_request( + model="gpt-5.1", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert result.get("store") is True + assert result.get("metadata") == {"key": "value"} + + +class TestDefaultParamValuesSafetyNet: + """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows + through completion() even without being a named parameter.""" + + def test_known_openai_param_excluded_from_non_default(self): + """should confirm get_non_default_completion_params excludes known OpenAI params""" + kwargs = {"store": True, "temperature": 0.5} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert "store" not in non_default + assert "temperature" not in non_default + + def test_unknown_param_included_in_non_default(self): + """should pass through unknown provider-specific params""" + kwargs = {"my_custom_provider_param": "foo"} + non_default = get_non_default_completion_params(kwargs=kwargs) + assert non_default.get("my_custom_provider_param") == "foo" + + def test_safety_net_forwards_recognized_kwargs(self): + """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + that are not already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"store": True, "metadata": {"key": "value"}} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["store"] is True + assert optional_param_args["metadata"] == {"key": "value"} + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_does_not_override_existing(self): + """should not override a param that's already in optional_param_args""" + optional_param_args = { + "model": "gpt-5.1", + "custom_llm_provider": "openai", + "temperature": 0.7, + } + kwargs = {"temperature": 0.9} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args["temperature"] == 0.7 + + def test_safety_net_skips_none_values(self): + """should not forward params with None value (the default)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": None} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert "store" not in optional_param_args + + def test_safety_net_forwards_falsy_non_none(self): + """should forward store=False (falsy but not None)""" + optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} + kwargs = {"store": False} + for k, v in kwargs.items(): + if ( + k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES + and k not in optional_param_args + and v is not None + ): + optional_param_args[k] = v + + assert optional_param_args.get("store") is False From 7542845e8db7cceffbbe3d0ada8f5360ec469800 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:53:19 +0530 Subject: [PATCH 08/20] Revert "fix: strip empty text content blocks in /v1/messages endpoint (#23097)" This reverts commit 2c738cc939c408cd0e85772bd503297c7d363197. --- litellm/llms/custom_httpx/llm_http_handler.py | 60 ----- ...est_v1_messages_empty_text_sanitization.py | 247 ------------------ 2 files changed, 307 deletions(-) delete mode 100644 tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6a5d669cad..1cef3e9ce1 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -152,59 +152,6 @@ else: LiteLLMLoggingObj = Any -def _sanitize_anthropic_messages_empty_text_blocks( - messages: List[Dict], -) -> List[Dict]: - """ - Strip empty text content blocks from Anthropic-format messages. - - Claude's API returns assistant messages with ``{"type": "text", "text": ""}`` - alongside ``tool_use`` blocks, but rejects them when sent back in subsequent - requests. This helper removes those empty text blocks so the /v1/messages - native path doesn't forward them as-is. - - - If a content list contains a mix of empty text blocks and other blocks - (e.g. tool_use), the empty text blocks are removed. - - If *all* blocks in a content list are empty text, the content is replaced - with a single non-empty placeholder to avoid sending an empty array. - - Ref: https://github.com/BerriAI/litellm/issues/22930 - """ - sanitized: List[Dict] = [] - for message in messages: - content = message.get("content") - if not isinstance(content, list): - sanitized.append(message) - continue - - filtered = [ - block - for block in content - if not ( - isinstance(block, dict) - and block.get("type") == "text" - and not block.get("text", "").strip() - ) - ] - - if filtered == content: - # Nothing was removed — keep original message as-is. - sanitized.append(message) - elif filtered: - # Some empty text blocks removed, but other content remains. - new_message = message.copy() - new_message["content"] = filtered - sanitized.append(new_message) - else: - # All blocks were empty text blocks. Replace with a placeholder - # so we don't send an empty content array. - new_message = message.copy() - new_message["content"] = [{"type": "text", "text": "..."}] - sanitized.append(new_message) - - return sanitized - - class BaseLLMHTTPHandler: async def _make_common_async_call( self, @@ -1958,13 +1905,6 @@ class BaseLLMHTTPHandler: anthropic_messages_optional_request_params, path ) - # Sanitize empty text content blocks from messages before forwarding. - # Claude's API returns assistant messages with empty text blocks - # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects - # them when sent back. Strip these to prevent 400 errors. - # Ref: https://github.com/BerriAI/litellm/issues/22930 - messages = _sanitize_anthropic_messages_empty_text_blocks(messages) - # Prepare request body request_body = anthropic_messages_provider_config.transform_anthropic_messages_request( model=model, diff --git a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py b/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py deleted file mode 100644 index b397b5a484..0000000000 --- a/tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py +++ /dev/null @@ -1,247 +0,0 @@ -""" -Test empty text content block sanitization for the /v1/messages native path. - -The Anthropic API returns assistant messages with empty text blocks -({"type": "text", "text": ""}) alongside tool_use blocks, but rejects -them when sent back. The /v1/messages endpoint must strip these before -forwarding to providers. - -Ref: https://github.com/BerriAI/litellm/issues/22930 -""" - -import pytest - -from litellm.llms.custom_httpx.llm_http_handler import ( - _sanitize_anthropic_messages_empty_text_blocks, -) - - -class TestSanitizeAnthropicMessagesEmptyTextBlocks: - """Unit tests for _sanitize_anthropic_messages_empty_text_blocks.""" - - def test_strips_empty_text_alongside_tool_use(self): - """ - The most common case from the bug report: an assistant message - containing an empty text block next to a tool_use block. - """ - messages = [ - {"role": "user", "content": "Run the command."}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_xxx", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result) == 2 - assert result[0] == messages[0] # user message unchanged - # assistant content should only have the tool_use block - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - - def test_preserves_nonempty_text_blocks(self): - """Non-empty text blocks must not be removed.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me check that."}, - { - "type": "tool_use", - "id": "toolu_yyy", - "name": "Bash", - "input": {"command": "pwd"}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 2 - assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."} - - def test_whitespace_only_text_block_stripped(self): - """Whitespace-only text blocks should also be stripped.""" - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": " \n\t "}, - { - "type": "tool_use", - "id": "toolu_zzz", - "name": "Bash", - "input": {}, - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "tool_use" - - def test_all_empty_text_blocks_replaced_with_placeholder(self): - """ - If all content blocks are empty text, replace with a placeholder - to avoid sending an empty content array. - """ - messages = [ - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["type"] == "text" - assert result[0]["content"][0]["text"].strip() # must be non-empty - - def test_string_content_untouched(self): - """Messages with string content should pass through unchanged.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there!"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_no_content_key_untouched(self): - """Messages without a content key should pass through.""" - messages = [ - {"role": "user", "content": "Hello"}, - {"role": "assistant"}, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_user_message_content_list_also_sanitized(self): - """ - Empty text blocks should be stripped from user messages too, - not just assistant messages. - """ - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "actual question"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert len(result[0]["content"]) == 1 - assert result[0]["content"][0]["text"] == "actual question" - - def test_tool_result_content_blocks_untouched(self): - """ - tool_result content blocks should not be affected — only - {"type": "text", "text": ""} blocks are stripped. - """ - messages = [ - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_xxx", - "content": "", - }, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - assert result == messages - - def test_multiple_messages_mixed(self): - """End-to-end scenario with multiple messages, some needing sanitization.""" - messages = [ - {"role": "user", "content": "Run ls"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {"command": "ls"}, - }, - ], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_1", - "content": "file1.txt\nfile2.txt", - }, - ], - }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Here are the files:"}, - ], - }, - ] - - result = _sanitize_anthropic_messages_empty_text_blocks(messages) - - # First message: string content, unchanged - assert result[0] == messages[0] - # Second message: empty text stripped, only tool_use remains - assert len(result[1]["content"]) == 1 - assert result[1]["content"][0]["type"] == "tool_use" - # Third message: tool_result, unchanged - assert result[2] == messages[2] - # Fourth message: non-empty text, unchanged - assert result[3] == messages[3] - - def test_does_not_mutate_original_messages(self): - """The function should not modify the input list or its dicts.""" - original_content = [ - {"type": "text", "text": ""}, - { - "type": "tool_use", - "id": "toolu_1", - "name": "Bash", - "input": {}, - }, - ] - messages = [ - { - "role": "assistant", - "content": original_content, - }, - ] - - _sanitize_anthropic_messages_empty_text_blocks(messages) - - # Original message content should be unchanged - assert len(messages[0]["content"]) == 2 - assert messages[0]["content"][0] == {"type": "text", "text": ""} From 3f30f6a49c7456a7dc19b3539646f7fe3b97cc39 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 09:59:53 +0530 Subject: [PATCH 09/20] Revert "Fix logging tests" --- litellm/litellm_core_utils/redact_messages.py | 10 ---------- provider_endpoints_support.json | 18 ------------------ 2 files changed, 28 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ddeb24d04a..ad68f3851a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -123,16 +123,6 @@ def perform_redaction(model_call_details: dict, result): elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] - elif isinstance(_result, dict) and "choices" in _result: - # ModelResponse.model_dump() returns dict - redact choices in place - if isinstance(_result.get("choices"), list) and len(_result["choices"]) > 0: - choice = _result["choices"][0] - if isinstance(choice, dict) and "message" in choice: - msg = choice["message"] - if isinstance(msg, dict) and "content" in msg: - msg["content"] = "redacted-by-litellm" - if isinstance(msg, dict) and "audio" in msg: - msg["audio"] = None else: return {"text": "redacted-by-litellm"} return _result diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 0b3f87fbe0..b1d4d5a116 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,24 +458,6 @@ "interactions": true } }, - "charity_engine": { - "display_name": "Charity Engine (`charity_engine`)", - "url": "https://docs.litellm.ai/docs/providers/charity_engine", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": false, - "interactions": false - } - }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { From 504e66ccd4b9a1052be23472a56cd630d1df4bdc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 10:22:00 +0530 Subject: [PATCH 10/20] =?UTF-8?q?Revert=20"fix:=20forward=20recognized=20O?= =?UTF-8?q?penAI=20params=20from=20kwargs=20in=20completion()=20(#2?= =?UTF-8?q?=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit dd6f0d6c55179634b5a5b8d1f670d97c56891b6c. --- litellm/main.py | 8 - .../llms/openai/chat/test_store_param.py | 188 ------------------ 2 files changed, 196 deletions(-) delete mode 100644 tests/test_litellm/llms/openai/chat/test_store_param.py diff --git a/litellm/main.py b/litellm/main.py index e23baadb79..364519e1fe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -65,7 +65,6 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging from litellm.constants import ( - DEFAULT_CHAT_COMPLETION_PARAM_VALUES, DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) @@ -1488,13 +1487,6 @@ def completion( # type: ignore # noqa: PLR0915 "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), } - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v optional_params = get_optional_params( **optional_param_args, **non_default_params ) diff --git a/tests/test_litellm/llms/openai/chat/test_store_param.py b/tests/test_litellm/llms/openai/chat/test_store_param.py deleted file mode 100644 index 0fd4799dae..0000000000 --- a/tests/test_litellm/llms/openai/chat/test_store_param.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Tests for the `store` parameter being correctly forwarded to OpenAI. - -Related issue: https://github.com/BerriAI/litellm/issues/23087 - -The `store` parameter was listed in OPENAI_CHAT_COMPLETION_PARAMS and -DEFAULT_CHAT_COMPLETION_PARAM_VALUES but was silently dropped because -get_non_default_completion_params() excluded it (as a "known" param) -while optional_param_args didn't include it (not a named param of -completion()). The fix adds a safety net in completion() that forwards -any kwargs present in DEFAULT_CHAT_COMPLETION_PARAM_VALUES that aren't -already in optional_param_args. -""" - -import os -import sys - -sys.path.insert(0, os.path.abspath("../../../../..")) - -from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES -from litellm.utils import get_non_default_completion_params, get_optional_params - - -class TestStoreParamForwarding: - """Tests that `store` flows through the parameter processing pipeline.""" - - def test_store_true_forwarded_for_openai(self): - """should forward store=True for OpenAI models via kwargs""" - result = get_optional_params( - model="gpt-5.1", - custom_llm_provider="openai", - store=True, - ) - assert result.get("store") is True - - def test_store_false_forwarded_for_openai(self): - """should forward store=False for OpenAI models""" - result = get_optional_params( - model="gpt-4o", - custom_llm_provider="openai", - store=False, - ) - assert result.get("store") is False - - def test_store_none_not_forwarded(self): - """should not include store when it is None (default)""" - result = get_optional_params( - model="gpt-4o", - custom_llm_provider="openai", - ) - assert "store" not in result - - def test_store_with_gpt5_models(self): - """should forward store=True for GPT-5 family models""" - for model in ["gpt-5.1", "gpt-5.2"]: - result = get_optional_params( - model=model, - custom_llm_provider="openai", - store=True, - ) - assert result.get("store") is True, f"store not forwarded for {model}" - - def test_store_in_supported_params(self): - """should list store as a supported OpenAI param""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - for model in ["gpt-4o", "gpt-5.1", "gpt-5.2"]: - supported = config.get_supported_openai_params(model) - assert "store" in supported, f"store not in supported params for {model}" - - def test_store_in_transform_request(self): - """should include store in the final transformed request body""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - messages = [{"role": "user", "content": "Hello"}] - optional_params = {"store": True} - result = config.transform_request( - model="gpt-5.1", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - assert result.get("store") is True - - def test_store_true_with_metadata(self): - """should forward both store and metadata when both are set""" - from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig - - config = OpenAIGPTConfig() - messages = [{"role": "user", "content": "Hello"}] - optional_params = {"store": True, "metadata": {"key": "value"}} - result = config.transform_request( - model="gpt-5.1", - messages=messages, - optional_params=optional_params, - litellm_params={}, - headers={}, - ) - assert result.get("store") is True - assert result.get("metadata") == {"key": "value"} - - -class TestDefaultParamValuesSafetyNet: - """Tests that any param in DEFAULT_CHAT_COMPLETION_PARAM_VALUES flows - through completion() even without being a named parameter.""" - - def test_known_openai_param_excluded_from_non_default(self): - """should confirm get_non_default_completion_params excludes known OpenAI params""" - kwargs = {"store": True, "temperature": 0.5} - non_default = get_non_default_completion_params(kwargs=kwargs) - assert "store" not in non_default - assert "temperature" not in non_default - - def test_unknown_param_included_in_non_default(self): - """should pass through unknown provider-specific params""" - kwargs = {"my_custom_provider_param": "foo"} - non_default = get_non_default_completion_params(kwargs=kwargs) - assert non_default.get("my_custom_provider_param") == "foo" - - def test_safety_net_forwards_recognized_kwargs(self): - """should forward kwargs in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - that are not already in optional_param_args""" - optional_param_args = { - "model": "gpt-5.1", - "custom_llm_provider": "openai", - "temperature": 0.7, - } - kwargs = {"store": True, "metadata": {"key": "value"}} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args["store"] is True - assert optional_param_args["metadata"] == {"key": "value"} - assert optional_param_args["temperature"] == 0.7 - - def test_safety_net_does_not_override_existing(self): - """should not override a param that's already in optional_param_args""" - optional_param_args = { - "model": "gpt-5.1", - "custom_llm_provider": "openai", - "temperature": 0.7, - } - kwargs = {"temperature": 0.9} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args["temperature"] == 0.7 - - def test_safety_net_skips_none_values(self): - """should not forward params with None value (the default)""" - optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} - kwargs = {"store": None} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert "store" not in optional_param_args - - def test_safety_net_forwards_falsy_non_none(self): - """should forward store=False (falsy but not None)""" - optional_param_args = {"model": "gpt-5.1", "custom_llm_provider": "openai"} - kwargs = {"store": False} - for k, v in kwargs.items(): - if ( - k in DEFAULT_CHAT_COMPLETION_PARAM_VALUES - and k not in optional_param_args - and v is not None - ): - optional_param_args[k] = v - - assert optional_param_args.get("store") is False From b08445837bd7fef2f2adf996dfd33db031b0aa3a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 10:21:49 +0530 Subject: [PATCH 11/20] fix(logging): preserve ModelResponse choices format in redacted standard_logging_object + add Charity Engine provider endpoint - Fix perform_redaction to handle dict representation of ModelResponse (from model_dump()) - Preserve full choices structure when redacting, redact content/audio in place - Add _redact_standard_logging_object helper for standard_logging_object field - Update test_logging_redaction_e2e_test assertions to expect choices format - Add charity_engine to provider_endpoints_support.json Fixes: test_standard_logging_payload, test_standard_logging_payload_audio Made-with: Cursor --- litellm/litellm_core_utils/redact_messages.py | 70 +++++++++++++++++++ provider_endpoints_support.json | 18 +++++ .../test_logging_redaction_e2e_test.py | 15 ++-- 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ad68f3851a..41cc200141 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -73,6 +73,53 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, dict) and "choices" in response: + # ModelResponse dict format - redact content in choices + if isinstance(response.get("choices"), list): + for choice in response["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # For other formats (empty dict, None, etc.), use simple text format + standard_logging_object["response"] = {"text": redacted_str} + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -114,6 +161,29 @@ def perform_redaction(model_call_details: dict, result): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) + elif isinstance(_result, dict) and "choices" in _result: + # Handle dict representation of ModelResponse (e.g., from model_dump()) + if _result.get("choices") is not None: + for choice in _result["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["message"]: + choice["message"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["message"]: + choice["message"]["thinking_blocks"] = None + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["delta"]: + choice["delta"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["delta"]: + choice["delta"]["thinking_blocks"] = None + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + else: + _redact_choice_content(choice) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 0536ec7205..0391a5a895 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -45,7 +45,8 @@ async def test_global_redaction_on(): await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload", @@ -75,7 +76,8 @@ async def test_global_redaction_with_dynamic_params(turn_off_message_logging): ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -108,7 +110,8 @@ async def test_global_redaction_off_with_dynamic_params(turn_off_message_logging json.dumps(standard_logging_payload, indent=2), ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -390,7 +393,8 @@ async def test_redaction_with_streaming_response(): assert standard_logging_payload is not None # Verify that redaction worked without pickle errors - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload for streaming with coroutine handling", @@ -477,5 +481,6 @@ async def test_redaction_with_metadata_completion_api(): # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, # the system checks the appropriate field for headers - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" From c8297332009e3c95a41960659593c67bd2607408 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:30:07 -0700 Subject: [PATCH 12/20] [Fix] Include model access groups when expanding All Proxy Models When a team has "all-proxy-models", the model list expansion now includes model access group names so they appear in the UI key creation form. Also fixes get_key_models not forwarding include_model_access_groups to _get_models_from_access_groups, and removes unused _unfurl_all_proxy_models. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 10 +- .../management_endpoints/team_endpoints.py | 18 --- .../proxy/auth/test_model_checks.py | 104 ++++++++++++++++++ 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 32f209a763..4ca1449208 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -112,10 +112,14 @@ def get_key_models( if SpecialModelNames.all_team_models.value in all_models: all_models = user_api_key_dict.team_models if SpecialModelNames.all_proxy_models.value in all_models: - all_models = proxy_model_list + all_models = list(proxy_model_list) # copy to avoid mutating caller's list + if include_model_access_groups: + all_models.extend(model_access_groups.keys()) all_models = _get_models_from_access_groups( - model_access_groups=model_access_groups, all_models=all_models + model_access_groups=model_access_groups, + all_models=all_models, + include_model_access_groups=include_model_access_groups, ) verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) @@ -141,6 +145,8 @@ def get_team_models( all_models_set.update(team_models) if SpecialModelNames.all_proxy_models.value in all_models_set: all_models_set.update(proxy_model_list) + if include_model_access_groups: + all_models_set.update(model_access_groups.keys()) all_models = list(all_models_set) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 633de86aa6..ee1868fc74 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2827,21 +2827,6 @@ async def validate_membership( ) -def _unfurl_all_proxy_models( - team_info: LiteLLM_TeamTable, llm_router: Router -) -> LiteLLM_TeamTable: - if ( - SpecialModelNames.all_proxy_models.value in team_info.models - and llm_router is not None - ): - team_models: set[str] = set() # make set to avoid duplicates - for model in team_info.models: - if model != SpecialModelNames.all_proxy_models.value: - team_models.add(model) - for model in llm_router.get_model_names(): - team_models.add(model) - team_info.models = list(team_models) - return team_info async def _add_team_member_budget_table( @@ -2972,9 +2957,6 @@ async def team_info( team_info_response_object=_team_info, ) - # ## UNFURL 'all-proxy-models' into the team_info.models list ## - # if llm_router is not None: - # _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 193b014f03..739ff25b7d 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -21,6 +21,110 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) +def test_get_team_models_all_proxy_models_includes_access_groups(): + """ + When a team has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names (e.g. 'claude-model-group') + in addition to individual model names. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=True + ) + assert "group-a" in result + assert "group-b" in result + assert "model1" in result + assert "model2" in result + + +def test_get_team_models_all_proxy_models_without_include_flag(): + """ + When include_model_access_groups=False, access group names should NOT + appear in the result even with 'all-proxy-models'. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=False + ) + assert "group-a" not in result + assert "group-b" not in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_all_proxy_models_includes_access_groups(): + """ + When a key has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["all-proxy-models"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_passes_include_model_access_groups(): + """ + When a key explicitly has an access group name in its models list and + include_model_access_groups=True, the group name should be retained + (not stripped by _get_models_from_access_groups). + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["group-a"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1", "model2"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ From 1cf191d9ad3a0732126d67b33813625419adafad Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:44:45 -0700 Subject: [PATCH 13/20] [Fix] Deduplicate model lists and remove dead assignment Adds dedup to get_key_models and get_team_models to prevent duplicate entries when access group member models overlap with proxy_model_list. Removes dead assignment of all_models in get_team_models. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 8 ++++++-- tests/test_litellm/proxy/auth/test_model_checks.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 4ca1449208..ccbc2f0194 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -122,6 +122,9 @@ def get_key_models( include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) return all_models @@ -148,14 +151,15 @@ def get_team_models( if include_model_access_groups: all_models_set.update(model_access_groups.keys()) - all_models = list(all_models_set) - all_models = _get_models_from_access_groups( model_access_groups=model_access_groups, all_models=list(all_models_set), include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL TEAM MODELS - {}".format(len(all_models))) return all_models diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 739ff25b7d..2b484bf975 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -43,6 +43,7 @@ def test_get_team_models_all_proxy_models_includes_access_groups(): assert "group-b" in result assert "model1" in result assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" def test_get_team_models_all_proxy_models_without_include_flag(): @@ -94,6 +95,7 @@ def test_get_key_models_all_proxy_models_includes_access_groups(): assert "group-a" in result assert "model1" in result assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" def test_get_key_models_passes_include_model_access_groups(): From 1755a281bd619eadbdad5501254694724ab78ae9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 22:55:53 -0700 Subject: [PATCH 14/20] Fix mutation bug: copy lists in get_key_models to prevent corrupting cached UserAPIKeyAuth `all_models = user_api_key_dict.models` was creating an alias, so `_get_models_from_access_groups` (which uses `.pop()`/`.extend()`) would mutate the cached object in-place. Now both `.models` and `.team_models` assignments create copies via `list()`. Added test to verify the input is not mutated. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/auth/model_checks.py | 4 +-- .../proxy/auth/test_model_checks.py | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index ccbc2f0194..13b26eef43 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -108,9 +108,9 @@ def get_key_models( """ all_models: List[str] = [] if len(user_api_key_dict.models) > 0: - all_models = user_api_key_dict.models + all_models = list(user_api_key_dict.models) # copy to avoid mutating cached objects if SpecialModelNames.all_team_models.value in all_models: - all_models = user_api_key_dict.team_models + all_models = list(user_api_key_dict.team_models) # copy to avoid mutating cached objects if SpecialModelNames.all_proxy_models.value in all_models: all_models = list(proxy_model_list) # copy to avoid mutating caller's list if include_model_access_groups: diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 2b484bf975..c43621d7f7 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -127,6 +127,34 @@ def test_get_key_models_passes_include_model_access_groups(): assert "model2" in result +def test_get_key_models_does_not_mutate_input(): + """ + get_key_models must not mutate user_api_key_dict.models in-place. + _get_models_from_access_groups uses .pop()/.extend() which would corrupt + cached UserAPIKeyAuth objects if all_models were an alias instead of a copy. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + original_models = ["group-a", "extra-model"] + user_api_key_dict = UserAPIKeyAuth( + models=list(original_models), # give it a list + api_key="test-key", + ) + model_access_groups = { + "group-a": ["model1", "model2"], + } + + _ = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=["model1", "model2"], + model_access_groups=model_access_groups, + include_model_access_groups=False, + ) + # The original models list on the auth object must be unchanged + assert user_api_key_dict.models == original_models + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ From db99fdeff3ab664b4292aa3c8a8c19d147c7162e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 11:34:23 +0530 Subject: [PATCH 15/20] fix(mcp): OpenAPI tool listing and execution for relative URLs and camelCase - Fix case-insensitive tool name matching in _tool_name_matches() so that OpenAPI operationIds (camelCase) match lowercase registered tool names when filtering by allowed_tools - Fix get_base_url() to resolve relative server URLs (e.g. /api/v3) by deriving full base URL from spec_path when OpenAPI spec has relative URLs - Add tests for case-insensitive matching and filter_tools_by_allowed_tools Made-with: Cursor --- .../mcp_server/openapi_to_mcp_generator.py | 19 ++- .../proxy/_experimental/mcp_server/server.py | 11 +- .../mcp_server/test_mcp_server.py | 147 ++++++++++++++++++ 3 files changed, 172 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 5f6cb87b26..5ad3cf444f 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -92,7 +92,24 @@ def get_base_url(spec: Dict[str, Any], spec_path: Optional[str] = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: - return spec["servers"][0]["url"] + server_url = spec["servers"][0]["url"] + + # If the server URL is relative (starts with /), derive base from spec_path + if server_url.startswith("/") and spec_path: + if spec_path.startswith("http://") or spec_path.startswith("https://"): + # Extract base URL from spec_path (e.g., https://petstore3.swagger.io/api/v3/openapi.json) + # Combine domain with the relative server URL + from urllib.parse import urlparse + parsed = urlparse(spec_path) + base_domain = f"{parsed.scheme}://{parsed.netloc}" + full_base_url = base_domain + server_url + verbose_logger.info( + f"OpenAPI spec has relative server URL '{server_url}'. " + f"Deriving base from spec_path: {full_base_url}" + ) + return full_base_url + + return server_url # OpenAPI 2.x (Swagger) elif "host" in spec: scheme = spec.get("schemes", ["https"])[0] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 99f6a5234a..7898f03e01 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -711,6 +711,7 @@ if MCP_AVAILABLE: Checks both the full tool name and unprefixed version (without server prefix). This allows users to configure simple tool names regardless of prefixing. + Comparison is case-insensitive to handle OpenAPI operationIds that may be in camelCase. Args: tool_name: The tool name to check (may be prefixed like "server-tool_name") @@ -723,13 +724,15 @@ if MCP_AVAILABLE: split_server_prefix_from_name, ) - # Check if the full name is in the list - if tool_name in filter_list: + # Normalize filter list to lowercase for case-insensitive comparison + filter_list_lower = [f.lower() for f in filter_list] + + if tool_name.lower() in filter_list_lower: return True - # Check if the unprefixed name is in the list + # Check if the unprefixed name is in the list (case-insensitive) unprefixed_name, _ = split_server_prefix_from_name(tool_name) - return unprefixed_name in filter_list + return unprefixed_name.lower() in filter_list_lower def filter_tools_by_allowed_tools( tools: List[MCPTool], diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index de2ec13b4a..a104ac2257 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2093,3 +2093,150 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["tool_count_total"] == 1 assert spend_meta["allowed_server_count"] == 1 assert spend_meta["per_server_tool_counts"]["server_a"] == 1 + + +def test_tool_name_matches_case_insensitive(): + """Test that _tool_name_matches performs case-insensitive comparison. + + This is critical for OpenAPI-based MCP servers where: + 1. operationIds are often in camelCase (e.g., 'addPet', 'updatePet') + 2. Tool names are lowercased during registration (e.g., 'addpet', 'updatepet') + 3. allowed_tools configuration may use the original camelCase names + + Without case-insensitive matching, all tools would be filtered out. + """ + try: + from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + except ImportError: + pytest.skip("MCP server not available") + + # Test case 1: Unprefixed tool name with camelCase in filter list + assert _tool_name_matches("addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("deletepet", ["addPet", "updatePet"]) is False + + # Test case 2: Prefixed tool name with camelCase in filter list + assert _tool_name_matches("per_store-addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-deletepet", ["addPet", "updatePet"]) is False + + # Test case 3: Mixed case variations + assert _tool_name_matches("findPetsByStatus", ["findpetsbystatus"]) is True + assert _tool_name_matches("findpetsbystatus", ["findPetsByStatus"]) is True + assert _tool_name_matches("FINDPETSBYSTATUS", ["findPetsByStatus"]) is True + + # Test case 4: Full prefixed name in filter list (case-insensitive) + assert _tool_name_matches("server-addPet", ["server-addpet"]) is True + assert _tool_name_matches("server-addpet", ["server-addPet"]) is True + + # Test case 5: Ensure non-matching names still don't match + assert _tool_name_matches("addpet", ["deletePet", "updatePet"]) is False + assert _tool_name_matches("server-addpet", ["deletePet", "updatePet"]) is False + + +def test_filter_tools_by_allowed_tools_case_insensitive(): + """Test that filter_tools_by_allowed_tools handles case-insensitive matching. + + Ensures that OpenAPI tools with lowercase names can be filtered using + camelCase allowed_tools configuration from the OpenAPI spec. + """ + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + # Create mock tools with lowercase names (as registered from OpenAPI) + tools = [ + MCPTool( + name="per_store-addpet", + description="Add a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-updatepet", + description="Update a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-deletepet", + description="Delete a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-findpetsbystatus", + description="Find pets by status", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Create mock server with camelCase allowed_tools (as from OpenAPI spec) + server = MCPServer( + server_id="test-server", + name="per_store", + transport=MCPTransport.http, + allowed_tools=["addPet", "updatePet", "findPetsByStatus"], + ) + + # Filter tools + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return 3 tools (case-insensitive match) + assert len(filtered_tools) == 3 + assert any(t.name == "per_store-addpet" for t in filtered_tools) + assert any(t.name == "per_store-updatepet" for t in filtered_tools) + assert any(t.name == "per_store-findpetsbystatus" for t in filtered_tools) + assert not any(t.name == "per_store-deletepet" for t in filtered_tools) + + +def test_filter_tools_by_allowed_tools_no_filter(): + """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + tools = [ + MCPTool( + name="fusion_litellm_mcp-model_list", + description="List models", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="fusion_litellm_mcp-chat_completion", + description="Chat completion", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Server with no allowed_tools filter + server = MCPServer( + server_id="test-server", + name="fusion_litellm_mcp", + transport=MCPTransport.http, + allowed_tools=None, + ) + + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return all tools when no filter is configured + assert len(filtered_tools) == 2 From 200b001633610341f0c032103ea52c40ca2daf7f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 10 Mar 2026 11:53:36 +0530 Subject: [PATCH 16/20] fix(bedrock): strip output_config from Converse requests; fix spend tracking redaction test Made-with: Cursor --- .../bedrock/chat/converse_transformation.py | 1 + .../chat/test_converse_transformation.py | 27 +++++++++++++++++++ .../test_spend_tracking_utils.py | 5 ++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d210f294c6..4fa407701c 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1206,6 +1206,7 @@ class AmazonConverseConfig(BaseConfig): self._validate_request_metadata(request_metadata) output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) + inference_params.pop("output_config", None) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 345f3ae7c5..7e1f235c49 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3170,6 +3170,33 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" +def test_output_config_snake_case_stripped_from_bedrock_converse_request(): + """Test that output_config (snake_case) is stripped from Bedrock Converse requests. + + Bedrock Converse API doesn't support the output_config parameter (Anthropic-only). + Nova and other Converse models reject requests with extraneous output_config. + """ + config = AmazonConverseConfig() + messages = [{"role": "user", "content": "test"}] + optional_params = { + "output_config": {"effort": "high"}, + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # output_config must not appear in additionalModelRequestFields + additional = result.get("additionalModelRequestFields", {}) + assert "output_config" not in additional, ( + f"output_config should be stripped for Bedrock Converse, got: {list(additional.keys())}" + ) + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9a64e641b5..3249a7ec79 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1071,9 +1071,10 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), - # perform_redaction returns {"text": "redacted-by-litellm"} + # perform_redaction redacts content in-place within the choices structure parsed_response = json.loads(response_result) - assert parsed_response == {"text": "redacted-by-litellm"} + assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert parsed_response["choices"][0]["message"]["role"] == "assistant" @patch("litellm.secret_managers.main.get_secret_bool") From 9ee489863d3e9f37ea805a09155e61dd86a16d9e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 9 Mar 2026 23:30:55 -0700 Subject: [PATCH 17/20] [Feature] UI - Virtual Keys: Add refetch button and keep stale data during refetch Show a Fetch/Fetching button next to "Showing X of Y results" that acts as both a manual refetch trigger and a loading indicator. The "Loading keys..." message now only appears on initial load; subsequent refetches keep the table visible with stale data (via React Query's keepPreviousData). Co-Authored-By: Claude Opus 4.6 --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 66 ++++++++++++++++++- .../VirtualKeysPage/VirtualKeysTable.tsx | 53 ++++++++++----- 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 4fd513b0d2..d7322d6d76 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -262,8 +262,8 @@ it("should display user email correctly", async () => { }); }); -it("should show skeleton loaders when isLoading is true", () => { - // Mock loading state +it("should show loading message only on initial load (isPending)", () => { + // Mock initial loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -283,7 +283,7 @@ it("should show skeleton loaders when isLoading is true", () => { renderWithProviders(); - // Check that loading message is shown + // Check that loading message is shown on initial load expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -795,3 +795,63 @@ describe("pagination display – total count and page count", () => { }); }); }); + +describe("refetch button", () => { + it("should show Fetch button in normal state", () => { + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeInTheDocument(); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); + + it("should show Fetching state and keep table data visible during refetch", () => { + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + // Button should show "Fetching" and be disabled + expect(screen.getByText("Fetching")).toBeInTheDocument(); + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeDisabled(); + + // Table data should still be visible (stale data) + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + + // "Loading keys..." should NOT appear during refetch + expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); + }); + + it("should call refetch when Fetch button is clicked", () => { + const mockRefetch = vi.fn(); + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: false, + refetch: mockRefetch, + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + fireEvent.click(fetchButton); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index fe9d58b979..b4588a899d 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -24,9 +24,9 @@ import { TableRow, Text, } from "@tremor/react"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip } from "antd"; -import React, { useEffect, useMemo, useState } from "react"; +import { InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; +import { Button as AntButton, Popover, Skeleton, Tooltip } from "antd"; +import React, { useEffect, useDeferredValue, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { useFilterLogic } from "../key_team_helpers/filter_logic"; import { PaginatedKeyAliasSelect } from "../KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect"; @@ -97,6 +97,15 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); + // Defer the transition so the button stays in loading state until the table + // has rendered with the new data (mirrors the spend-logs pattern) + const isFetchingDeferred = useDeferredValue(isFetching); + const isButtonLoading = isFetching || isFetchingDeferred; + + const handleRefresh = () => { + refetch(); + }; + const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -606,16 +615,28 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading || isFetching ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} +
+ {isLoading ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )} + + } + onClick={handleRefresh} + disabled={isButtonLoading} + title="Fetch data" + > + {isButtonLoading ? "Fetching" : "Fetch"} + +
- {isLoading || isFetching ? ( + {isLoading ? ( ) : ( @@ -623,24 +644,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : (