From b6fcda2f8a77743022f67d43cc769536434065ca Mon Sep 17 00:00:00 2001 From: "Jugal D. Bhatt" <55304795+jugaldb@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:46:47 -0700 Subject: [PATCH] [LLM Translation] Fix model group on clientside auth with API calls (#13314) * fix unsupported operand type(s) for +=: 'NoneType' and 'str' on clientside auth creds for responses * fix the client side auth to use correct metadata * add more tests * fix tests --- litellm/router.py | 9 +- tests/local_testing/test_router_utils.py | 1 + .../test_router_helper_utils.py | 228 ++++++++++++++++++ 3 files changed, 235 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 406a85297c..3be88596b1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1406,7 +1406,7 @@ class Router: kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults) def _handle_clientside_credential( - self, deployment: dict, kwargs: dict + self, deployment: dict, kwargs: dict, function_name: Optional[str] = None ) -> Deployment: """ Handle clientside credential @@ -1417,7 +1417,10 @@ class Router: litellm_params=litellm_params, request_kwargs=kwargs ) # Use deployment model_name as model_group for generating model_id - model_group = deployment["model_name"] + metadata_variable_name = _get_router_metadata_variable_name( + function_name=function_name, + ) + model_group = kwargs.get(metadata_variable_name, {}).get("model_group") _model_id = self._generate_model_id( model_group=model_group, litellm_params=dynamic_litellm_params ) @@ -1451,7 +1454,7 @@ class Router: deployment_model_name = deployment["model_name"] if is_clientside_credential(request_kwargs=kwargs): deployment_pydantic_obj = self._handle_clientside_credential( - deployment=deployment, kwargs=kwargs + deployment=deployment, kwargs=kwargs, function_name=function_name ) model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index cd26f8ad60..f3b0238be5 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -414,6 +414,7 @@ def test_router_handle_clientside_credential(): "api_key": "123", "metadata": {"model_group": "gemini/gemini-1.5-flash"}, }, + function_name="acompletion", ) assert new_deployment.litellm_params.api_key == "123" diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 915a7b11ae..07a3253095 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1441,3 +1441,231 @@ def test_handle_clientside_credential_with_deployment_model_name(model_list): print("✓ _handle_clientside_credential test passed!") + +@pytest.mark.parametrize("function_name, expected_metadata_key", [ + ("acompletion", "metadata"), + ("_ageneric_api_call_with_fallbacks", "litellm_metadata"), + ("batch", "litellm_metadata"), + ("completion", "metadata"), + ("acreate_file", "litellm_metadata"), + ("aget_file", "litellm_metadata"), +]) +def test_handle_clientside_credential_metadata_loading(model_list, function_name, expected_metadata_key): + """Test that _handle_clientside_credential correctly loads metadata based on function name""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-123" + } + } + + # Mock kwargs with clientside credentials and metadata + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + expected_metadata_key: { + "model_group": "gpt-4.1", + "custom_field": "test_value" + } + } + + # Call the function + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name=function_name + ) + + # Verify the result is a Deployment object + assert isinstance(result_deployment, Deployment) + + # Verify the deployment has the correct model_name (should be the model_group from metadata) + assert result_deployment.model_name == "gpt-4.1" + + # Verify the litellm_params contain the clientside credentials + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + + # Verify the model_info has been updated with a new ID + assert result_deployment.model_info.id != "original-id-123" + assert result_deployment.model_info.original_model_id == "original-id-123" + + # Verify the deployment was added to the router + assert len(router.model_list) == len(model_list) + 1 + + # Test that the function correctly uses the right metadata key + # For acompletion, it should use "metadata" + # For _ageneric_api_call_with_fallbacks/batch, it should use "litellm_metadata" + if function_name == "acompletion": + assert "metadata" in kwargs + assert "litellm_metadata" not in kwargs + elif function_name in ["_ageneric_api_call_with_fallbacks", "batch", "acreate_file", "aget_file"]: + assert "litellm_metadata" in kwargs + # Note: acompletion would not have litellm_metadata, but other functions might have both + + print(f"✓ Success with function_name '{function_name}' using '{expected_metadata_key}' metadata key") + + +@pytest.mark.parametrize("function_name, metadata_key", [ + ("acompletion", "metadata"), + ("_ageneric_api_call_with_fallbacks", "litellm_metadata"), +]) +def test_handle_clientside_credential_metadata_variable_name(model_list, function_name, metadata_key): + """Test that _handle_clientside_credential uses the correct metadata variable name based on function name""" + from litellm.router_utils.batch_utils import _get_router_metadata_variable_name + + router = Router(model_list=model_list) + + # Verify the metadata variable name is correct for each function + expected_metadata_key = _get_router_metadata_variable_name(function_name=function_name) + assert expected_metadata_key == metadata_key + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-456" + } + } + + # Mock kwargs with clientside credentials and the correct metadata key + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + metadata_key: { + "model_group": "gpt-4.1", + "test_field": "test_value" + } + } + + # Call the function + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name=function_name + ) + + # Verify the function correctly extracted model_group from the right metadata key + assert result_deployment.model_name == "gpt-4.1" + + # Verify the deployment was created with the correct metadata + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + + print(f"✓ Success with function_name '{function_name}' correctly using '{metadata_key}' for metadata") + + +def test_handle_clientside_credential_no_metadata(model_list): + """Test that _handle_clientside_credential handles cases where no metadata is provided""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-789" + } + } + + # Mock kwargs with clientside credentials but NO metadata + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1" + # No metadata key at all + } + + # This should fail because there's no model_group in metadata + # The function expects to find model_group in the metadata + try: + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name="acompletion" + ) + # If we get here, the function should have used deployment.model_name as fallback + assert result_deployment.model_name == "gpt-4.1" + print("✓ Success with no metadata - used deployment.model_name as fallback") + except Exception as e: + # This is expected behavior - the function needs model_group to generate model_id + print(f"✓ Correctly handled no metadata case: {e}") + + # Test with empty metadata + kwargs_with_empty_metadata = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + "metadata": {} # Empty metadata + } + + try: + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs_with_empty_metadata, + function_name="acompletion" + ) + # Should fail because empty metadata has no model_group + pytest.fail("Expected failure with empty metadata") + except Exception as e: + print(f"✓ Correctly handled empty metadata case: {e}") + + +def test_handle_clientside_credential_with_responses_function(model_list): + """Test that _handle_clientside_credential works correctly with responses function name""" + router = Router(model_list=model_list) + + # Mock deployment + deployment = { + "model_name": "gpt-4.1", + "litellm_params": { + "model": "gpt-4.1", + "api_key": "test_key" + }, + "model_info": { + "id": "original-id-responses" + } + } + + # Mock kwargs with clientside credentials and litellm_metadata (for responses function) + kwargs = { + "api_key": "client_side_key", + "api_base": "https://api.openai.com/v1", + "litellm_metadata": { + "model_group": "gpt-4.1", + "responses_field": "responses_value" + } + } + + # Call the function with _ageneric_api_call_with_fallbacks function name (which handles responses) + result_deployment = router._handle_clientside_credential( + deployment=deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks" + ) + + # Verify the result + assert isinstance(result_deployment, Deployment) + assert result_deployment.model_name == "gpt-4.1" + assert result_deployment.litellm_params.api_key == "client_side_key" + assert result_deployment.litellm_params.api_base == "https://api.openai.com/v1" + assert result_deployment.model_info.id != "original-id-responses" + assert result_deployment.model_info.original_model_id == "original-id-responses" + + # Verify the deployment was added to the router + assert len(router.model_list) == len(model_list) + 1 + + print("✓ Success with _ageneric_api_call_with_fallbacks function name and litellm_metadata") +