diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d1aebe4dce..9e8a1585c4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -293,19 +293,20 @@ def _override_openai_response_model( we preserve the actual model that was used (the fallback model). 2. If the request was to an Azure Model Router, we preserve the actual model that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model. + 3. If this was a fastest_response batch completion, use the winning model's + model group name instead of the comma-separated list the client sent. """ if not requested_model: return - # Check if a fallback occurred - if so, preserve the actual model used hidden_params = getattr(response_obj, "_hidden_params", {}) or {} if isinstance(hidden_params, dict): + # Check if a fallback occurred - if so, preserve the actual model used fallback_headers = hidden_params.get("additional_headers", {}) or {} attempted_fallbacks = fallback_headers.get( "x-litellm-attempted-fallbacks", None ) if attempted_fallbacks is not None and attempted_fallbacks > 0: - # A fallback occurred - preserve the actual model that was used verbose_proxy_logger.debug( "%s: fallback detected (attempted_fallbacks=%d), preserving actual model used instead of overriding to requested model.", log_context, @@ -313,6 +314,25 @@ def _override_openai_response_model( ) return + # For fastest_response batch completions, use the winning model's group + # name rather than the comma-separated list the client sent. + if hidden_params.get("fastest_response_batch_completion"): + winning_model = fallback_headers.get("x-litellm-model-group") + if winning_model: + verbose_proxy_logger.debug( + "%s: fastest_response detected, using winning model group=%r instead of requested=%r.", + log_context, + winning_model, + requested_model, + ) + requested_model = winning_model + else: + verbose_proxy_logger.debug( + "%s: fastest_response detected but no model group header found, preserving actual model from response.", + log_context, + ) + return + # Check if this is an Azure Model Router request - if so, preserve the actual model used if _is_azure_model_router_request(requested_model): verbose_proxy_logger.debug( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 853ceb63c9..5cc1e0b4f3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5721,6 +5721,11 @@ def _restamp_streaming_chunk_model( if _is_azure_model_router_request(requested_model_from_client): return chunk, model_mismatch_logged + # For fastest_response batch completions, preserve the winning model's name + # instead of stamping the comma-separated list the client sent. + if request_data.get("fastest_response", False): + return chunk, model_mismatch_logged + downstream_model = ( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 223f0b335f..45ec4b726a 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1431,6 +1431,83 @@ class TestOverrideOpenAIResponseModel: assert response_obj.model == actual_model_used assert response_obj.model != requested_model + def test_override_model_uses_winning_model_for_fastest_response(self): + """ + Test that when fastest_response batch completion is used with a + comma-separated model list, the response model is set to the winning + model's group name (not the comma-separated list). + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model_group = "gemini/gemini-2.5-flash" + downstream_model = "gemini-2.5-flash" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": { + "x-litellm-model-group": winning_model_group, + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == winning_model_group + assert response_obj.model != requested_model + + def test_override_model_preserves_response_when_fastest_response_no_model_group( + self, + ): + """ + Test that when fastest_response is set but no model group header is + available, the actual downstream model is preserved. + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": {}, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == downstream_model + + def test_override_model_normal_when_fastest_response_not_set(self): + """ + Test that when fastest_response_batch_completion is not set, the + normal override behavior applies (model is set to requested_model). + """ + requested_model = "openai/gpt-4o" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "additional_headers": { + "x-litellm-model-group": "openai/gpt-4o", + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == requested_model + class TestIsAzureModelRouterRequest: """Tests for _is_azure_model_router_request helper""" diff --git a/tests/test_litellm/proxy/test_response_model_sanitization.py b/tests/test_litellm/proxy/test_response_model_sanitization.py index 22785bbcb9..b7253d9833 100644 --- a/tests/test_litellm/proxy/test_response_model_sanitization.py +++ b/tests/test_litellm/proxy/test_response_model_sanitization.py @@ -273,3 +273,61 @@ async def test_proxy_streaming_azure_model_router_preserves_actual_model(monkeyp # Azure Model Router: preserve actual model used, not the router model assert payload["model"] == actual_model_used assert payload["model"] != router_model + + +@pytest.mark.asyncio +async def test_proxy_streaming_fastest_response_preserves_winning_model(monkeypatch): + """ + Regression test for fastest_response streaming: + + When the client sends a comma-separated model list with fastest_response=True, + the streaming chunks should preserve the winning model's name from the + downstream response, NOT override to the comma-separated list. + """ + comma_separated_models = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model = "gemini-2.5-flash" + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + async def _iterator_hook( + user_api_key_dict: UserAPIKeyAuth, + response: AsyncGenerator, + request_data: dict, + ): + yield _make_model_response_stream_chunk(model=winning_model) + + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_iterator_hook", + _iterator_hook, + ) + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_hook", + AsyncMock(side_effect=lambda **kwargs: kwargs["response"]), + ) + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-1234") + + gen = proxy_server.async_data_generator( + response=MagicMock(), + user_api_key_dict=user_api_key_dict, + request_data={ + "model": comma_separated_models, + "_litellm_client_requested_model": comma_separated_models, + "fastest_response": True, + }, + ) + + chunks = [] + async for item in gen: + chunks.append(item) + + assert len(chunks) >= 2 + first = chunks[0] + assert first.startswith("data: ") + + payload = json.loads(first[len("data: ") :].strip()) + assert payload["model"] == winning_model + assert payload["model"] != comma_separated_models