From fa23218e756a889e727f0183eef13d133daa0f58 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 27 Mar 2026 11:53:45 -0700 Subject: [PATCH 1/2] docs: add faq to docs --- .../blog/security_townhall_updates/index.md | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/my-website/blog/security_townhall_updates/index.md b/docs/my-website/blog/security_townhall_updates/index.md index 6b2260156d..b997de9c18 100644 --- a/docs/my-website/blog/security_townhall_updates/index.md +++ b/docs/my-website/blog/security_townhall_updates/index.md @@ -24,12 +24,6 @@ On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.8 At this point, our understanding is that this was a supply-chain incident affecting those two published versions. -### Q. Were older packages impacted? - -Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs. - -We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) - ## How did this happen? Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline. @@ -160,6 +154,24 @@ We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for t We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c). +## Frequently Asked Questions + +**Q: Did you observe any lateral movement into your corporate environment during this incident?** + +A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation. + +**Q: Do you expect delays in future product releases due to these new security measures?** + +A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure. + +**Q: Were older packages impacted?** + +Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs. + +We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions) + + + ## Questions & Support If you believe your systems may be affected, contact us immediately: From 32adda8a49e488a327ebeaf8fe81f158f317cf97 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 27 Mar 2026 22:34:26 -0700 Subject: [PATCH 2/2] fix: return winning model name instead of comma-separated list for fastest_response When fastest_response=true with comma-separated models, the response model field was stamped with the entire comma-separated string. Now uses the x-litellm-model-group header from the winning response to return the correct model name. Made-with: Cursor --- litellm/proxy/common_request_processing.py | 24 +++++- litellm/proxy/proxy_server.py | 5 ++ .../proxy/test_common_request_processing.py | 77 +++++++++++++++++++ .../proxy/test_response_model_sanitization.py | 58 ++++++++++++++ 4 files changed, 162 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d1aebe4dce..9e8a1585c4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -293,19 +293,20 @@ def _override_openai_response_model( we preserve the actual model that was used (the fallback model). 2. If the request was to an Azure Model Router, we preserve the actual model that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model. + 3. If this was a fastest_response batch completion, use the winning model's + model group name instead of the comma-separated list the client sent. """ if not requested_model: return - # Check if a fallback occurred - if so, preserve the actual model used hidden_params = getattr(response_obj, "_hidden_params", {}) or {} if isinstance(hidden_params, dict): + # Check if a fallback occurred - if so, preserve the actual model used fallback_headers = hidden_params.get("additional_headers", {}) or {} attempted_fallbacks = fallback_headers.get( "x-litellm-attempted-fallbacks", None ) if attempted_fallbacks is not None and attempted_fallbacks > 0: - # A fallback occurred - preserve the actual model that was used verbose_proxy_logger.debug( "%s: fallback detected (attempted_fallbacks=%d), preserving actual model used instead of overriding to requested model.", log_context, @@ -313,6 +314,25 @@ def _override_openai_response_model( ) return + # For fastest_response batch completions, use the winning model's group + # name rather than the comma-separated list the client sent. + if hidden_params.get("fastest_response_batch_completion"): + winning_model = fallback_headers.get("x-litellm-model-group") + if winning_model: + verbose_proxy_logger.debug( + "%s: fastest_response detected, using winning model group=%r instead of requested=%r.", + log_context, + winning_model, + requested_model, + ) + requested_model = winning_model + else: + verbose_proxy_logger.debug( + "%s: fastest_response detected but no model group header found, preserving actual model from response.", + log_context, + ) + return + # Check if this is an Azure Model Router request - if so, preserve the actual model used if _is_azure_model_router_request(requested_model): verbose_proxy_logger.debug( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7d3d2ceb53..eb8411d80e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5516,6 +5516,11 @@ def _restamp_streaming_chunk_model( if _is_azure_model_router_request(requested_model_from_client): return chunk, model_mismatch_logged + # For fastest_response batch completions, preserve the winning model's name + # instead of stamping the comma-separated list the client sent. + if request_data.get("fastest_response", False): + return chunk, model_mismatch_logged + downstream_model = ( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 223f0b335f..45ec4b726a 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1431,6 +1431,83 @@ class TestOverrideOpenAIResponseModel: assert response_obj.model == actual_model_used assert response_obj.model != requested_model + def test_override_model_uses_winning_model_for_fastest_response(self): + """ + Test that when fastest_response batch completion is used with a + comma-separated model list, the response model is set to the winning + model's group name (not the comma-separated list). + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model_group = "gemini/gemini-2.5-flash" + downstream_model = "gemini-2.5-flash" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": { + "x-litellm-model-group": winning_model_group, + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == winning_model_group + assert response_obj.model != requested_model + + def test_override_model_preserves_response_when_fastest_response_no_model_group( + self, + ): + """ + Test that when fastest_response is set but no model group header is + available, the actual downstream model is preserved. + """ + requested_model = "openai/gpt-4o,gemini/gemini-2.5-flash" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "fastest_response_batch_completion": True, + "additional_headers": {}, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == downstream_model + + def test_override_model_normal_when_fastest_response_not_set(self): + """ + Test that when fastest_response_batch_completion is not set, the + normal override behavior applies (model is set to requested_model). + """ + requested_model = "openai/gpt-4o" + downstream_model = "gpt-4o-2024-08-06" + + response_obj = MagicMock() + response_obj.model = downstream_model + response_obj._hidden_params = { + "additional_headers": { + "x-litellm-model-group": "openai/gpt-4o", + }, + } + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + + assert response_obj.model == requested_model + class TestIsAzureModelRouterRequest: """Tests for _is_azure_model_router_request helper""" diff --git a/tests/test_litellm/proxy/test_response_model_sanitization.py b/tests/test_litellm/proxy/test_response_model_sanitization.py index 22785bbcb9..b7253d9833 100644 --- a/tests/test_litellm/proxy/test_response_model_sanitization.py +++ b/tests/test_litellm/proxy/test_response_model_sanitization.py @@ -273,3 +273,61 @@ async def test_proxy_streaming_azure_model_router_preserves_actual_model(monkeyp # Azure Model Router: preserve actual model used, not the router model assert payload["model"] == actual_model_used assert payload["model"] != router_model + + +@pytest.mark.asyncio +async def test_proxy_streaming_fastest_response_preserves_winning_model(monkeypatch): + """ + Regression test for fastest_response streaming: + + When the client sends a comma-separated model list with fastest_response=True, + the streaming chunks should preserve the winning model's name from the + downstream response, NOT override to the comma-separated list. + """ + comma_separated_models = "openai/gpt-4o,gemini/gemini-2.5-flash" + winning_model = "gemini-2.5-flash" + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + async def _iterator_hook( + user_api_key_dict: UserAPIKeyAuth, + response: AsyncGenerator, + request_data: dict, + ): + yield _make_model_response_stream_chunk(model=winning_model) + + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_iterator_hook", + _iterator_hook, + ) + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_hook", + AsyncMock(side_effect=lambda **kwargs: kwargs["response"]), + ) + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-1234") + + gen = proxy_server.async_data_generator( + response=MagicMock(), + user_api_key_dict=user_api_key_dict, + request_data={ + "model": comma_separated_models, + "_litellm_client_requested_model": comma_separated_models, + "fastest_response": True, + }, + ) + + chunks = [] + async for item in gen: + chunks.append(item) + + assert len(chunks) >= 2 + first = chunks[0] + assert first.startswith("data: ") + + payload = json.loads(first[len("data: ") :].strip()) + assert payload["model"] == winning_model + assert payload["model"] != comma_separated_models