diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ad68f3851a..41cc200141 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -73,6 +73,53 @@ def _redact_responses_api_output(output_items): summary_item.text = "redacted-by-litellm" +def _redact_standard_logging_object(model_call_details: dict): + """Redact messages and response inside standard_logging_object if present.""" + standard_logging_object = model_call_details.get("standard_logging_object") + if standard_logging_object is None: + return + + redacted_str = "redacted-by-litellm" + + if standard_logging_object.get("messages") is not None: + standard_logging_object["messages"] = [ + {"role": "user", "content": redacted_str} + ] + + response = standard_logging_object.get("response") + if response is not None: + if isinstance(response, dict) and "output" in response: + # ResponsesAPIResponse format - redact content in output items + if isinstance(response.get("output"), list): + for output_item in response["output"]: + if isinstance(output_item, dict) and "content" in output_item: + if isinstance(output_item["content"], list): + for content_item in output_item["content"]: + if ( + isinstance(content_item, dict) + and "text" in content_item + ): + content_item["text"] = redacted_str + elif isinstance(response, dict) and "choices" in response: + # ModelResponse dict format - redact content in choices + if isinstance(response.get("choices"), list): + for choice in response["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = redacted_str + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = redacted_str + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + elif isinstance(response, str): + standard_logging_object["response"] = redacted_str + else: + # For other formats (empty dict, None, etc.), use simple text format + standard_logging_object["response"] = {"text": redacted_str} + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -114,6 +161,29 @@ def perform_redaction(model_call_details: dict, result): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) + elif isinstance(_result, dict) and "choices" in _result: + # Handle dict representation of ModelResponse (e.g., from model_dump()) + if _result.get("choices") is not None: + for choice in _result["choices"]: + if isinstance(choice, dict): + if "message" in choice and isinstance(choice["message"], dict): + choice["message"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["message"]: + choice["message"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["message"]: + choice["message"]["thinking_blocks"] = None + if "audio" in choice["message"]: + choice["message"]["audio"] = None + elif "delta" in choice and isinstance(choice["delta"], dict): + choice["delta"]["content"] = "redacted-by-litellm" + if "reasoning_content" in choice["delta"]: + choice["delta"]["reasoning_content"] = "redacted-by-litellm" + if "thinking_blocks" in choice["delta"]: + choice["delta"]["thinking_blocks"] = None + if "audio" in choice["delta"]: + choice["delta"]["audio"] = None + else: + _redact_choice_content(choice) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d210f294c6..4fa407701c 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1206,6 +1206,7 @@ class AmazonConverseConfig(BaseConfig): self._validate_request_metadata(request_metadata) output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) + inference_params.pop("output_config", None) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' additional_request_params = { diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b3125d4ad3..275c352b39 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -94,5 +94,12 @@ "assemblyai": { "base_url": "https://llm-gateway.assemblyai.com/v1", "api_key_env": "ASSEMBLYAI_API_KEY" + }, + "charity_engine": { + "base_url": "https://api.charityengine.services/remotejobs/v2/inference", + "api_key_env": "CHARITY_ENGINE_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 5f6cb87b26..5ad3cf444f 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -92,7 +92,24 @@ def get_base_url(spec: Dict[str, Any], spec_path: Optional[str] = None) -> str: """Extract base URL from OpenAPI spec.""" # OpenAPI 3.x if "servers" in spec and spec["servers"]: - return spec["servers"][0]["url"] + server_url = spec["servers"][0]["url"] + + # If the server URL is relative (starts with /), derive base from spec_path + if server_url.startswith("/") and spec_path: + if spec_path.startswith("http://") or spec_path.startswith("https://"): + # Extract base URL from spec_path (e.g., https://petstore3.swagger.io/api/v3/openapi.json) + # Combine domain with the relative server URL + from urllib.parse import urlparse + parsed = urlparse(spec_path) + base_domain = f"{parsed.scheme}://{parsed.netloc}" + full_base_url = base_domain + server_url + verbose_logger.info( + f"OpenAPI spec has relative server URL '{server_url}'. " + f"Deriving base from spec_path: {full_base_url}" + ) + return full_base_url + + return server_url # OpenAPI 2.x (Swagger) elif "host" in spec: scheme = spec.get("schemes", ["https"])[0] diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 99f6a5234a..7898f03e01 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -711,6 +711,7 @@ if MCP_AVAILABLE: Checks both the full tool name and unprefixed version (without server prefix). This allows users to configure simple tool names regardless of prefixing. + Comparison is case-insensitive to handle OpenAPI operationIds that may be in camelCase. Args: tool_name: The tool name to check (may be prefixed like "server-tool_name") @@ -723,13 +724,15 @@ if MCP_AVAILABLE: split_server_prefix_from_name, ) - # Check if the full name is in the list - if tool_name in filter_list: + # Normalize filter list to lowercase for case-insensitive comparison + filter_list_lower = [f.lower() for f in filter_list] + + if tool_name.lower() in filter_list_lower: return True - # Check if the unprefixed name is in the list + # Check if the unprefixed name is in the list (case-insensitive) unprefixed_name, _ = split_server_prefix_from_name(tool_name) - return unprefixed_name in filter_list + return unprefixed_name.lower() in filter_list_lower def filter_tools_by_allowed_tools( tools: List[MCPTool], diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 32f209a763..13b26eef43 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -108,16 +108,23 @@ def get_key_models( """ all_models: List[str] = [] if len(user_api_key_dict.models) > 0: - all_models = user_api_key_dict.models + all_models = list(user_api_key_dict.models) # copy to avoid mutating cached objects if SpecialModelNames.all_team_models.value in all_models: - all_models = user_api_key_dict.team_models + all_models = list(user_api_key_dict.team_models) # copy to avoid mutating cached objects if SpecialModelNames.all_proxy_models.value in all_models: - all_models = proxy_model_list + all_models = list(proxy_model_list) # copy to avoid mutating caller's list + if include_model_access_groups: + all_models.extend(model_access_groups.keys()) all_models = _get_models_from_access_groups( - model_access_groups=model_access_groups, all_models=all_models + model_access_groups=model_access_groups, + all_models=all_models, + include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL KEY MODELS - {}".format(len(all_models))) return all_models @@ -141,8 +148,8 @@ def get_team_models( all_models_set.update(team_models) if SpecialModelNames.all_proxy_models.value in all_models_set: all_models_set.update(proxy_model_list) - - all_models = list(all_models_set) + if include_model_access_groups: + all_models_set.update(model_access_groups.keys()) all_models = _get_models_from_access_groups( model_access_groups=model_access_groups, @@ -150,6 +157,9 @@ def get_team_models( include_model_access_groups=include_model_access_groups, ) + # deduplicate while preserving order + all_models = list(dict.fromkeys(all_models)) + verbose_proxy_logger.debug("ALL TEAM MODELS - {}".format(len(all_models))) return all_models diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 633de86aa6..ee1868fc74 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -2827,21 +2827,6 @@ async def validate_membership( ) -def _unfurl_all_proxy_models( - team_info: LiteLLM_TeamTable, llm_router: Router -) -> LiteLLM_TeamTable: - if ( - SpecialModelNames.all_proxy_models.value in team_info.models - and llm_router is not None - ): - team_models: set[str] = set() # make set to avoid duplicates - for model in team_info.models: - if model != SpecialModelNames.all_proxy_models.value: - team_models.add(model) - for model in llm_router.get_model_names(): - team_models.add(model) - team_info.models = list(team_models) - return team_info async def _add_team_member_budget_table( @@ -2972,9 +2957,6 @@ async def team_info( team_info_response_object=_team_info, ) - # ## UNFURL 'all-proxy-models' into the team_info.models list ## - # if llm_router is not None: - # _team_info = _unfurl_all_proxy_models(_team_info, llm_router) response_object = TeamInfoResponseObject( team_id=team_id, team_info=_team_info, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8ae0cf2892..b5d5c06924 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3177,6 +3177,7 @@ class LlmProviders(str, Enum): TOPAZ = "topaz" SAP_GENERATIVE_AI_HUB = "sap" ASSEMBLYAI = "assemblyai" + CHARITY_ENGINE = "charity_engine" GITHUB_COPILOT = "github_copilot" SNOWFLAKE = "snowflake" GRADIENT_AI = "gradient_ai" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b1d4d5a116..0b3f87fbe0 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -458,6 +458,24 @@ "interactions": true } }, + "charity_engine": { + "display_name": "Charity Engine (`charity_engine`)", + "url": "https://docs.litellm.ai/docs/providers/charity_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "chutes": { "display_name": "Chutes (`chutes`)", "endpoints": { diff --git a/tests/llm_translation/test_skills_api.py b/tests/llm_translation/test_skills_api.py index 7565ba7440..b340167133 100644 --- a/tests/llm_translation/test_skills_api.py +++ b/tests/llm_translation/test_skills_api.py @@ -44,19 +44,25 @@ def create_skill_zip(skill_name: str, unique_suffix: Optional[str] = None): skill_dir = test_dir / skill_name # Create a zip file containing the skill directory + # When unique_suffix is set, folder name must match skill name in SKILL.md (Anthropic requirement) + zip_folder_name = f"{skill_name}-{unique_suffix}" if unique_suffix else skill_name zip_path = test_dir / f"{skill_name}.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: - zf.write(skill_dir, arcname=skill_name) - if unique_suffix is not None: - # Rewrite SKILL.md with a unique name to avoid API conflicts + # Rewrite SKILL.md with a unique name and use matching folder name skill_md = (skill_dir / "SKILL.md").read_text() skill_md = skill_md.replace( f"name: {skill_name}", - f"name: {skill_name}-{unique_suffix}", + f"name: {zip_folder_name}", ) - zf.writestr(f"{skill_name}/SKILL.md", skill_md) + zf.writestr(f"{zip_folder_name}/SKILL.md", skill_md) + # Add any other files in the skill dir (e.g. subdirs) under the new folder name + for f in skill_dir.rglob("*"): + if f.is_file() and f.name != "SKILL.md": + rel = f.relative_to(skill_dir) + zf.write(f, arcname=f"{zip_folder_name}/{rel}") else: + zf.write(skill_dir, arcname=skill_name) zf.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md") try: diff --git a/tests/local_testing/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py index fcdfcfe6e7..ead387599d 100644 --- a/tests/local_testing/test_custom_callback_input.py +++ b/tests/local_testing/test_custom_callback_input.py @@ -1300,9 +1300,11 @@ def test_logging_async_cache_hit_sync_call(turn_off_message_logging): "redacted-by-litellm" == standard_logging_object["messages"][0]["content"] ) - assert {"text": "redacted-by-litellm"} == standard_logging_object[ - "response" - ] + # response is a full ModelResponse dict (choices format) since d84e5e381acf + assert ( + standard_logging_object["response"]["choices"][0]["message"]["content"] + == "redacted-by-litellm" + ) def test_logging_standard_payload_failure_call(): diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 0536ec7205..0391a5a895 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -45,7 +45,8 @@ async def test_global_redaction_on(): await asyncio.sleep(1) standard_logging_payload = test_custom_logger.logged_standard_logging_payload assert standard_logging_payload is not None - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload", @@ -75,7 +76,8 @@ async def test_global_redaction_with_dynamic_params(turn_off_message_logging): ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -108,7 +110,8 @@ async def test_global_redaction_off_with_dynamic_params(turn_off_message_logging json.dumps(standard_logging_payload, indent=2), ) if turn_off_message_logging is True: - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert ( standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" ) @@ -390,7 +393,8 @@ async def test_redaction_with_streaming_response(): assert standard_logging_payload is not None # Verify that redaction worked without pickle errors - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" print( "logged standard logging payload for streaming with coroutine handling", @@ -477,5 +481,6 @@ async def test_redaction_with_metadata_completion_api(): # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, # the system checks the appropriate field for headers - assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + response = standard_logging_payload["response"] + assert response["choices"][0]["message"]["content"] == "redacted-by-litellm" assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 345f3ae7c5..7e1f235c49 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -3170,6 +3170,33 @@ def test_transform_request_with_output_config(): assert result["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "TestSchema" +def test_output_config_snake_case_stripped_from_bedrock_converse_request(): + """Test that output_config (snake_case) is stripped from Bedrock Converse requests. + + Bedrock Converse API doesn't support the output_config parameter (Anthropic-only). + Nova and other Converse models reject requests with extraneous output_config. + """ + config = AmazonConverseConfig() + messages = [{"role": "user", "content": "test"}] + optional_params = { + "output_config": {"effort": "high"}, + } + + result = config._transform_request( + model="us.amazon.nova-pro-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + # output_config must not appear in additionalModelRequestFields + additional = result.get("additionalModelRequestFields", {}) + assert "output_config" not in additional, ( + f"output_config should be stripped for Bedrock Converse, got: {list(additional.keys())}" + ) + + def test_transform_response_native_structured_output(): """Test response handling when model returns JSON as text content (native structured output).""" response_json = { diff --git a/tests/test_litellm/llms/openai_like/test_charity_engine.py b/tests/test_litellm/llms/openai_like/test_charity_engine.py new file mode 100644 index 0000000000..5d6a751b62 --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_charity_engine.py @@ -0,0 +1,101 @@ +""" +Tests for Charity Engine provider configuration and integration. +""" + +import os +import sys + +try: + import pytest +except ImportError: + pytest = None + +# Add workspace to path +workspace_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +sys.path.insert(0, workspace_path) + +import litellm + + +class TestCharityEngineProviderConfig: + """Test Charity Engine provider configuration""" + + def test_charity_engine_in_provider_list(self): + """Test that charity_engine is in the provider list""" + from litellm import LlmProviders + + assert hasattr(LlmProviders, "CHARITY_ENGINE") + assert LlmProviders.CHARITY_ENGINE.value == "charity_engine" + assert "charity_engine" in litellm.provider_list + + def test_charity_engine_json_config_exists(self): + """Test that charity_engine is configured in providers.json""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("charity_engine") + + charity_engine = JSONProviderRegistry.get("charity_engine") + assert charity_engine is not None + assert charity_engine.base_url == "https://api.charityengine.services/remotejobs/v2/inference" + assert charity_engine.api_key_env == "CHARITY_ENGINE_API_KEY" + assert charity_engine.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_charity_engine_provider_resolution(self): + """Test that provider resolution finds charity_engine""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="charity_engine/gemma3:270m", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "gemma3:270m" + assert provider == "charity_engine" + assert api_base == "https://api.charityengine.services/remotejobs/v2/inference" + + def test_charity_engine_router_config(self): + """Test that charity_engine can be used in Router configuration""" + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "gemma3-270m", + "litellm_params": { + "model": "charity_engine/gemma3:270m", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "gemma3-270m" + + +if __name__ == "__main__": + print("Testing Charity Engine Provider...") + + test_config = TestCharityEngineProviderConfig() + + print("\n1. Testing provider in list...") + test_config.test_charity_engine_in_provider_list() + print(" ✓ charity_engine in provider list") + + print("\n2. Testing JSON config...") + test_config.test_charity_engine_json_config_exists() + print(" ✓ charity_engine JSON config loaded") + + print("\n3. Testing provider resolution...") + test_config.test_charity_engine_provider_resolution() + print(" ✓ Provider resolution works") + + print("\n4. Testing router configuration...") + test_config.test_charity_engine_router_config() + print(" ✓ Router configuration works") + + print("\n" + "=" * 50) + print("✓ All configuration tests passed!") + print("=" * 50) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index de2ec13b4a..a104ac2257 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -2093,3 +2093,150 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert spend_meta["tool_count_total"] == 1 assert spend_meta["allowed_server_count"] == 1 assert spend_meta["per_server_tool_counts"]["server_a"] == 1 + + +def test_tool_name_matches_case_insensitive(): + """Test that _tool_name_matches performs case-insensitive comparison. + + This is critical for OpenAPI-based MCP servers where: + 1. operationIds are often in camelCase (e.g., 'addPet', 'updatePet') + 2. Tool names are lowercased during registration (e.g., 'addpet', 'updatepet') + 3. allowed_tools configuration may use the original camelCase names + + Without case-insensitive matching, all tools would be filtered out. + """ + try: + from litellm.proxy._experimental.mcp_server.server import _tool_name_matches + except ImportError: + pytest.skip("MCP server not available") + + # Test case 1: Unprefixed tool name with camelCase in filter list + assert _tool_name_matches("addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("deletepet", ["addPet", "updatePet"]) is False + + # Test case 2: Prefixed tool name with camelCase in filter list + assert _tool_name_matches("per_store-addpet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-updatepet", ["addPet", "updatePet"]) is True + assert _tool_name_matches("per_store-deletepet", ["addPet", "updatePet"]) is False + + # Test case 3: Mixed case variations + assert _tool_name_matches("findPetsByStatus", ["findpetsbystatus"]) is True + assert _tool_name_matches("findpetsbystatus", ["findPetsByStatus"]) is True + assert _tool_name_matches("FINDPETSBYSTATUS", ["findPetsByStatus"]) is True + + # Test case 4: Full prefixed name in filter list (case-insensitive) + assert _tool_name_matches("server-addPet", ["server-addpet"]) is True + assert _tool_name_matches("server-addpet", ["server-addPet"]) is True + + # Test case 5: Ensure non-matching names still don't match + assert _tool_name_matches("addpet", ["deletePet", "updatePet"]) is False + assert _tool_name_matches("server-addpet", ["deletePet", "updatePet"]) is False + + +def test_filter_tools_by_allowed_tools_case_insensitive(): + """Test that filter_tools_by_allowed_tools handles case-insensitive matching. + + Ensures that OpenAPI tools with lowercase names can be filtered using + camelCase allowed_tools configuration from the OpenAPI spec. + """ + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + # Create mock tools with lowercase names (as registered from OpenAPI) + tools = [ + MCPTool( + name="per_store-addpet", + description="Add a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-updatepet", + description="Update a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-deletepet", + description="Delete a pet", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="per_store-findpetsbystatus", + description="Find pets by status", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Create mock server with camelCase allowed_tools (as from OpenAPI spec) + server = MCPServer( + server_id="test-server", + name="per_store", + transport=MCPTransport.http, + allowed_tools=["addPet", "updatePet", "findPetsByStatus"], + ) + + # Filter tools + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return 3 tools (case-insensitive match) + assert len(filtered_tools) == 3 + assert any(t.name == "per_store-addpet" for t in filtered_tools) + assert any(t.name == "per_store-updatepet" for t in filtered_tools) + assert any(t.name == "per_store-findpetsbystatus" for t in filtered_tools) + assert not any(t.name == "per_store-deletepet" for t in filtered_tools) + + +def test_filter_tools_by_allowed_tools_no_filter(): + """Test that filter_tools_by_allowed_tools returns all tools when no filter is set.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + filter_tools_by_allowed_tools, + ) + from litellm.types.mcp_server.tool_registry import MCPTool + except ImportError: + pytest.skip("MCP server not available") + + # Mock handler function + def mock_handler(**kwargs): + return kwargs + + tools = [ + MCPTool( + name="fusion_litellm_mcp-model_list", + description="List models", + input_schema={"type": "object"}, + handler=mock_handler, + ), + MCPTool( + name="fusion_litellm_mcp-chat_completion", + description="Chat completion", + input_schema={"type": "object"}, + handler=mock_handler, + ), + ] + + # Server with no allowed_tools filter + server = MCPServer( + server_id="test-server", + name="fusion_litellm_mcp", + transport=MCPTransport.http, + allowed_tools=None, + ) + + filtered_tools = filter_tools_by_allowed_tools(tools, server) + + # Should return all tools when no filter is configured + assert len(filtered_tools) == 2 diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 193b014f03..c43621d7f7 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -21,6 +21,140 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) +def test_get_team_models_all_proxy_models_includes_access_groups(): + """ + When a team has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names (e.g. 'claude-model-group') + in addition to individual model names. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=True + ) + assert "group-a" in result + assert "group-b" in result + assert "model1" in result + assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" + + +def test_get_team_models_all_proxy_models_without_include_flag(): + """ + When include_model_access_groups=False, access group names should NOT + appear in the result even with 'all-proxy-models'. + """ + from litellm.proxy.auth.model_checks import get_team_models + + team_models = ["all-proxy-models"] + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + "group-b": ["model2"], + } + + result = get_team_models( + team_models, proxy_model_list, model_access_groups, include_model_access_groups=False + ) + assert "group-a" not in result + assert "group-b" not in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_all_proxy_models_includes_access_groups(): + """ + When a key has 'all-proxy-models' and include_model_access_groups=True, + the result should include model access group names. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["all-proxy-models"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + assert len(result) == len(set(result)), "result should have no duplicates" + + +def test_get_key_models_passes_include_model_access_groups(): + """ + When a key explicitly has an access group name in its models list and + include_model_access_groups=True, the group name should be retained + (not stripped by _get_models_from_access_groups). + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + user_api_key_dict = UserAPIKeyAuth( + models=["group-a"], + api_key="test-key", + ) + proxy_model_list = ["model1", "model2"] + model_access_groups = { + "group-a": ["model1", "model2"], + } + + result = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + assert "group-a" in result + assert "model1" in result + assert "model2" in result + + +def test_get_key_models_does_not_mutate_input(): + """ + get_key_models must not mutate user_api_key_dict.models in-place. + _get_models_from_access_groups uses .pop()/.extend() which would corrupt + cached UserAPIKeyAuth objects if all_models were an alias instead of a copy. + """ + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.model_checks import get_key_models + + original_models = ["group-a", "extra-model"] + user_api_key_dict = UserAPIKeyAuth( + models=list(original_models), # give it a list + api_key="test-key", + ) + model_access_groups = { + "group-a": ["model1", "model2"], + } + + _ = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=["model1", "model2"], + model_access_groups=model_access_groups, + include_model_access_groups=False, + ) + # The original models list on the auth object must be unchanged + assert user_api_key_dict.models == original_models + + @pytest.mark.parametrize( "key_models,team_models,proxy_model_list,model_list,expected", [ diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9a64e641b5..3249a7ec79 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -1071,9 +1071,10 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) # When redaction is enabled and response is a dict (not ModelResponse), - # perform_redaction returns {"text": "redacted-by-litellm"} + # perform_redaction redacts content in-place within the choices structure parsed_response = json.loads(response_result) - assert parsed_response == {"text": "redacted-by-litellm"} + assert parsed_response["choices"][0]["message"]["content"] == "redacted-by-litellm" + assert parsed_response["choices"][0]["message"]["role"] == "assistant" @patch("litellm.secret_managers.main.get_secret_bool") diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 4fd513b0d2..35f87e8770 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -262,8 +262,8 @@ it("should display user email correctly", async () => { }); }); -it("should show skeleton loaders when isLoading is true", () => { - // Mock loading state +it("should show loading message only on initial load (isPending)", () => { + // Mock initial loading state mockUseKeys.mockReturnValue({ data: null, isPending: true, @@ -283,7 +283,7 @@ it("should show skeleton loaders when isLoading is true", () => { renderWithProviders(); - // Check that loading message is shown + // Check that loading message is shown on initial load expect(screen.getByText("🚅 Loading keys...")).toBeInTheDocument(); // Check that actual key data is not shown @@ -795,3 +795,79 @@ describe("pagination display – total count and page count", () => { }); }); }); + +describe("refetch button", () => { + it("should show Fetch button in normal state", () => { + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeInTheDocument(); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); + + it("should show Fetching state and keep table data visible during refetch", () => { + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + // Button should show "Fetching" and be disabled + expect(screen.getByText("Fetching")).toBeInTheDocument(); + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).toBeDisabled(); + + // Table data should still be visible (stale data) + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); + + // "Loading keys..." should NOT appear during refetch + expect(screen.queryByText("🚅 Loading keys...")).not.toBeInTheDocument(); + }); + + it("should call refetch when Fetch button is clicked", () => { + const mockRefetch = vi.fn(); + mockUseKeys.mockReturnValue({ + data: { + keys: [mockKey], + total_count: 1, + current_page: 1, + total_pages: 1, + } as KeysResponse, + isPending: false, + isFetching: false, + refetch: mockRefetch, + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + fireEvent.click(fetchButton); + + expect(mockRefetch).toHaveBeenCalledTimes(1); + }); + + it("should show Fetch button enabled on error so user can retry", () => { + mockUseKeys.mockReturnValue({ + data: null, + isPending: false, + isFetching: false, + isError: true, + refetch: vi.fn(), + } as any); + + renderWithProviders(); + + const fetchButton = screen.getByTitle("Fetch data"); + expect(fetchButton).not.toBeDisabled(); + expect(screen.getByText("Fetch")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index fe9d58b979..d9c590938f 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -24,9 +24,9 @@ import { TableRow, Text, } from "@tremor/react"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Popover, Skeleton, Tooltip } from "antd"; -import React, { useEffect, useMemo, useState } from "react"; +import { InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; +import { Button as AntButton, Popover, Skeleton, Tooltip } from "antd"; +import React, { useEffect, useDeferredValue, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { useFilterLogic } from "../key_team_helpers/filter_logic"; import { PaginatedKeyAliasSelect } from "../KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect"; @@ -81,6 +81,7 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo data: keys, isPending: isLoading, isFetching, + isError, refetch, } = useKeys(tablePagination.pageIndex + 1, tablePagination.pageSize, { sortBy: sortBy || undefined, @@ -97,6 +98,15 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo organizations, }); + // Defer the transition so the button stays in loading state until the table + // has rendered with the new data (mirrors the spend-logs pattern) + const isFetchingDeferred = useDeferredValue(isFetching); + const isButtonLoading = (isFetching || isFetchingDeferred) && !isError; + + const handleRefresh = () => { + refetch(); + }; + const totalCount = filteredTotalCount ?? keys?.total_count ?? 0; // Add a useEffect to call refresh when a key is created @@ -606,16 +616,28 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading || isFetching ? ( - - ) : ( - - Showing {rangeLabel} of {totalCount} results - - )} +
+ {isLoading ? ( + + ) : ( + + Showing {rangeLabel} of {totalCount} results + + )} + + } + onClick={handleRefresh} + disabled={isButtonLoading} + title="Fetch data" + > + {isButtonLoading ? "Fetching" : "Fetch"} + +
- {isLoading || isFetching ? ( + {isLoading ? ( ) : ( @@ -623,24 +645,24 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : ( )} - {isLoading || isFetching ? ( + {isLoading ? ( ) : (