Merge pull request #24753 from BerriAI/litellm_dev_03_27_2026_p1

Fix returned model when batch completions is used - return picked model, not comma-separated list

litellm/proxy/common_request_processing.py

@@ -293,19 +293,20 @@ def _override_openai_response_model(
        we preserve the actual model that was used (the fallback model).
     2. If the request was to an Azure Model Router, we preserve the actual model
        that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model.
+    3. If this was a fastest_response batch completion, use the winning model's
+       model group name instead of the comma-separated list the client sent.
     """
     if not requested_model:
         return
 
-    # Check if a fallback occurred - if so, preserve the actual model used
     hidden_params = getattr(response_obj, "_hidden_params", {}) or {}
     if isinstance(hidden_params, dict):
+        # Check if a fallback occurred - if so, preserve the actual model used
         fallback_headers = hidden_params.get("additional_headers", {}) or {}
         attempted_fallbacks = fallback_headers.get(
             "x-litellm-attempted-fallbacks", None
         )
         if attempted_fallbacks is not None and attempted_fallbacks > 0:
-            # A fallback occurred - preserve the actual model that was used
             verbose_proxy_logger.debug(
                 "%s: fallback detected (attempted_fallbacks=%d), preserving actual model used instead of overriding to requested model.",
                 log_context,
@@ -313,6 +314,25 @@ def _override_openai_response_model(
             )
             return
 
+        # For fastest_response batch completions, use the winning model's group
+        # name rather than the comma-separated list the client sent.
+        if hidden_params.get("fastest_response_batch_completion"):
+            winning_model = fallback_headers.get("x-litellm-model-group")
+            if winning_model:
+                verbose_proxy_logger.debug(
+                    "%s: fastest_response detected, using winning model group=%r instead of requested=%r.",
+                    log_context,
+                    winning_model,
+                    requested_model,
+                )
+                requested_model = winning_model
+            else:
+                verbose_proxy_logger.debug(
+                    "%s: fastest_response detected but no model group header found, preserving actual model from response.",
+                    log_context,
+                )
+                return
+
     # Check if this is an Azure Model Router request - if so, preserve the actual model used
     if _is_azure_model_router_request(requested_model):
         verbose_proxy_logger.debug(

litellm/proxy/proxy_server.py

@@ -5721,6 +5721,11 @@ def _restamp_streaming_chunk_model(
     if _is_azure_model_router_request(requested_model_from_client):
         return chunk, model_mismatch_logged
 
+    # For fastest_response batch completions, preserve the winning model's name
+    # instead of stamping the comma-separated list the client sent.
+    if request_data.get("fastest_response", False):
+        return chunk, model_mismatch_logged
+
     downstream_model = (
         chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None)
     )

tests/test_litellm/proxy/test_common_request_processing.py

@@ -1431,6 +1431,83 @@ class TestOverrideOpenAIResponseModel:
         assert response_obj.model == actual_model_used
         assert response_obj.model != requested_model
 
+    def test_override_model_uses_winning_model_for_fastest_response(self):
+        """
+        Test that when fastest_response batch completion is used with a
+        comma-separated model list, the response model is set to the winning
+        model's group name (not the comma-separated list).
+        """
+        requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash"
+        winning_model_group = "gemini/gemini-2.5-flash"
+        downstream_model = "gemini-2.5-flash"
+
+        response_obj = MagicMock()
+        response_obj.model = downstream_model
+        response_obj._hidden_params = {
+            "fastest_response_batch_completion": True,
+            "additional_headers": {
+                "x-litellm-model-group": winning_model_group,
+            },
+        }
+
+        _override_openai_response_model(
+            response_obj=response_obj,
+            requested_model=requested_model,
+            log_context="test_context",
+        )
+
+        assert response_obj.model == winning_model_group
+        assert response_obj.model != requested_model
+
+    def test_override_model_preserves_response_when_fastest_response_no_model_group(
+        self,
+    ):
+        """
+        Test that when fastest_response is set but no model group header is
+        available, the actual downstream model is preserved.
+        """
+        requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash"
+        downstream_model = "gpt-4o-2024-08-06"
+
+        response_obj = MagicMock()
+        response_obj.model = downstream_model
+        response_obj._hidden_params = {
+            "fastest_response_batch_completion": True,
+            "additional_headers": {},
+        }
+
+        _override_openai_response_model(
+            response_obj=response_obj,
+            requested_model=requested_model,
+            log_context="test_context",
+        )
+
+        assert response_obj.model == downstream_model
+
+    def test_override_model_normal_when_fastest_response_not_set(self):
+        """
+        Test that when fastest_response_batch_completion is not set, the
+        normal override behavior applies (model is set to requested_model).
+        """
+        requested_model = "openai/gpt-4o"
+        downstream_model = "gpt-4o-2024-08-06"
+
+        response_obj = MagicMock()
+        response_obj.model = downstream_model
+        response_obj._hidden_params = {
+            "additional_headers": {
+                "x-litellm-model-group": "openai/gpt-4o",
+            },
+        }
+
+        _override_openai_response_model(
+            response_obj=response_obj,
+            requested_model=requested_model,
+            log_context="test_context",
+        )
+
+        assert response_obj.model == requested_model
+
 
 class TestIsAzureModelRouterRequest:
     """Tests for _is_azure_model_router_request helper"""

tests/test_litellm/proxy/test_response_model_sanitization.py

@@ -273,3 +273,61 @@ async def test_proxy_streaming_azure_model_router_preserves_actual_model(monkeyp
     # Azure Model Router: preserve actual model used, not the router model
     assert payload["model"] == actual_model_used
     assert payload["model"] != router_model
+
+
+@pytest.mark.asyncio
+async def test_proxy_streaming_fastest_response_preserves_winning_model(monkeypatch):
+    """
+    Regression test for fastest_response streaming:
+
+    When the client sends a comma-separated model list with fastest_response=True,
+    the streaming chunks should preserve the winning model's name from the
+    downstream response, NOT override to the comma-separated list.
+    """
+    comma_separated_models = "openai/gpt-4o,gemini/gemini-2.5-flash"
+    winning_model = "gemini-2.5-flash"
+
+    from litellm.proxy import proxy_server
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    async def _iterator_hook(
+        user_api_key_dict: UserAPIKeyAuth,
+        response: AsyncGenerator,
+        request_data: dict,
+    ):
+        yield _make_model_response_stream_chunk(model=winning_model)
+
+    monkeypatch.setattr(
+        proxy_server.proxy_logging_obj,
+        "async_post_call_streaming_iterator_hook",
+        _iterator_hook,
+    )
+    monkeypatch.setattr(
+        proxy_server.proxy_logging_obj,
+        "async_post_call_streaming_hook",
+        AsyncMock(side_effect=lambda **kwargs: kwargs["response"]),
+    )
+
+    user_api_key_dict = UserAPIKeyAuth(api_key="sk-1234")
+
+    gen = proxy_server.async_data_generator(
+        response=MagicMock(),
+        user_api_key_dict=user_api_key_dict,
+        request_data={
+            "model": comma_separated_models,
+            "_litellm_client_requested_model": comma_separated_models,
+            "fastest_response": True,
+        },
+    )
+
+    chunks = []
+    async for item in gen:
+        chunks.append(item)
+
+    assert len(chunks) >= 2
+    first = chunks[0]
+    assert first.startswith("data: ")
+
+    payload = json.loads(first[len("data: ") :].strip())
+    assert payload["model"] == winning_model
+    assert payload["model"] != comma_separated_models