From ed5dc6a0776ce96c6a6cfe990395f3aca44106d5 Mon Sep 17 00:00:00 2001 From: Alexandros Solanos Date: Tue, 16 Dec 2025 15:28:35 +0100 Subject: [PATCH 01/57] Improve model repetition detection performance --- .../litellm_core_utils/streaming_handler.py | 52 ++++---- .../test_streaming_handler.py | 117 ++++++++++++++++++ 2 files changed, 147 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index d92af41717..b8240839d7 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -145,6 +145,7 @@ class CustomStreamWrapper: self.chunks: List = ( [] ) # keep track of the returned chunks - used for calculating the input/output tokens for stream options + self._repeated_messages_count = 1 self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: Optional[int] = None @@ -190,7 +191,7 @@ class CustomStreamWrapper: except Exception as e: raise e - def safety_checker(self) -> None: + def raise_on_model_repetition(self) -> None: """ Fixes - https://github.com/BerriAI/litellm/issues/5158 @@ -198,29 +199,36 @@ class CustomStreamWrapper: Raises - InternalServerError, if LLM enters infinite loop while streaming """ - if len(self.chunks) >= litellm.REPEATED_STREAMING_CHUNK_LIMIT: - # Get the last n chunks - last_chunks = self.chunks[-litellm.REPEATED_STREAMING_CHUNK_LIMIT :] + if len(self.chunks) < 2: + return - # Extract the relevant content from the chunks - last_contents = [chunk.choices[0].delta.content for chunk in last_chunks] + last_content = self.chunks[-1].choices[0].delta.content - # Check if all extracted contents are identical - if all(content == last_contents[0] for content in last_contents): - if ( - last_contents[0] is not None - and isinstance(last_contents[0], str) - and len(last_contents[0]) > 2 - ): # ignore empty content - https://github.com/BerriAI/litellm/issues/5158#issuecomment-2287156946 - # All last n chunks are identical - raise litellm.InternalServerError( - message="The model is repeating the same chunk = {}.".format( - last_contents[0] - ), - model="", - llm_provider="", - ) + if ( + last_content is None + or not isinstance(last_content, str) + or len(last_content) <= 2 + ): # ignore empty content - https://github.com/BerriAI/litellm/issues/5158#issuecomment-2287156946 + self._repeated_messages_count = 1 + return + second_to_last_content = self.chunks[-2].choices[0].delta.content + + if last_content == second_to_last_content: + self._repeated_messages_count += 1 + else: + self._repeated_messages_count = 1 + + if self._repeated_messages_count >= litellm.REPEATED_STREAMING_CHUNK_LIMIT: + # All last n chunks are identical + raise litellm.InternalServerError( + message="The model is repeating the same chunk = {}.".format( + last_content + ), + model="", + llm_provider="", + ) + def check_special_tokens(self, chunk: str, finish_reason: Optional[str]): """ Output parse / special tokens for sagemaker + hf streaming. @@ -879,7 +887,7 @@ class CustomStreamWrapper: if ( is_chunk_non_empty ): # cannot set content of an OpenAI Object to be an empty string - self.safety_checker() + self.raise_on_model_repetition() hold, model_response_str = self.check_special_tokens( chunk=completion_obj["content"], finish_reason=model_response.choices[0].finish_reason, diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 6a528fef8f..fcce6ddc67 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1162,3 +1162,120 @@ def test_is_chunk_non_empty_with_valid_tool_calls( ) is True ) + + +def _make_chunk(content: Optional[str]) -> ModelResponseStream: + return ModelResponseStream( + id="test", + created=1741037890, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content=content))], + ) + + +def _build_chunks(pattern: list[str], N: int) -> list[ModelResponseStream]: + """ + Build a list of chunks based on a pattern specification. + """ + chunks = [] + for i, p in enumerate(pattern): + if p == "same": + chunks.append(_make_chunk("same_chunk")) + elif p == "diff": + chunks.append(_make_chunk(f"chunk_{i}")) + else: + chunks.append(_make_chunk(p)) + return chunks + +_REPETITION_TEST_CASES = [ + # Basic cases + pytest.param( + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + True, + id="all_identical_raises", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="below_threshold_no_raise", + ), + pytest.param( + [None] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="none_content_no_raise", + ), + pytest.param( + [""] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="empty_content_no_raise", + ), + # Short content (len <= 2) should not raise + pytest.param( + ["##"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_no_raise", + ), + pytest.param( + ["{"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_1char_no_raise", + ), + pytest.param( + ["ab"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="short_content_2chars_ab_no_raise", + ), + # All different chunks + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT, + False, + id="all_different_no_raise", + ), + # One chunk different at various positions + pytest.param( + ["different_first"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1), + False, + id="first_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 1) + ["different_last"], + False, + id="last_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1) + ["different_mid"] + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - litellm.REPEATED_STREAMING_CHUNK_LIMIT // 2 + 1), + False, + id="middle_chunk_different_no_raise", + ), + pytest.param( + ["same"] * (litellm.REPEATED_STREAMING_CHUNK_LIMIT - 2) + ["diff", "diff"], + False, + id="last_two_different_no_raise", + ), + pytest.param( + ["diff"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["same"] * litellm.REPEATED_STREAMING_CHUNK_LIMIT + ["diff"], + True, + id="in_between_same_and_diff_raise", + ), +] + + +@pytest.mark.parametrize("chunks_pattern,should_raise", _REPETITION_TEST_CASES) +def test_raise_on_model_repetition( + initialized_custom_stream_wrapper: CustomStreamWrapper, + chunks_pattern: list, + should_raise: bool, +): + wrapper = initialized_custom_stream_wrapper + chunks = _build_chunks(chunks_pattern, len(chunks_pattern)) + + if should_raise: + with pytest.raises(litellm.InternalServerError) as exc_info: + for chunk in chunks: + wrapper.chunks.append(chunk) + wrapper.raise_on_model_repetition() + assert "repeating the same chunk" in str(exc_info.value) + else: + for chunk in chunks: + wrapper.chunks.append(chunk) + wrapper.raise_on_model_repetition() \ No newline at end of file From 19efe556cbd5e52f4ad68414400b9e88f86706de Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:41:32 +0000 Subject: [PATCH 02/57] fix: /key/block and /key/unblock return 404 instead of misleading 401 for non-existent keys The block_key() and unblock_key() handlers previously returned a misleading 401 'Authentication Error' when the body 'key' didn't exist in the database, even though authentication (via Authorization header) succeeded correctly. Root cause: After auth passed, the handlers called get_key_object() for cache refresh. This function was designed for auth token lookup and raises ProxyException(code=401) when a token isn't found. Additionally, Prisma's update() silently returns None for non-existent records instead of raising an error, so the code reached get_key_object() without detecting the missing key. Fix: - Add an explicit existence check (find_unique) before the update - Return 404 ProxyException with 'Key not found' if the key doesn't exist - Replace get_key_object() + manual cache update with _delete_cache_key_object() to invalidate the cache (next read will re-fetch from DB) - Reuse the find_unique result for audit logs, eliminating duplicate queries Co-authored-by: yuneng-jiang --- dev_config.yaml | 9 +- .../key_management_endpoints.py | 90 +++++++------------ 2 files changed, 33 insertions(+), 66 deletions(-) diff --git a/dev_config.yaml b/dev_config.yaml index 64e3c14703..142e0bf94e 100644 --- a/dev_config.yaml +++ b/dev_config.yaml @@ -1,13 +1,8 @@ model_list: - - model_name: fake-openai-endpoint + - model_name: gpt-4 litellm_params: - model: openai/fake-model + model: gpt-4 api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 - -litellm_settings: - drop_params: True - telemetry: False diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 1c0c212b60..6dd0d4137b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4788,18 +4788,19 @@ async def block_key( else: hashed_token = data.key - if litellm.store_audit_logs is True: - # make an audit log for key update - record = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + # Check if the key exists before trying to block it + existing_record = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} + ) + if existing_record is None: + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) - if record is None: - raise ProxyException( - message=f"Key {data.key} not found", - type=ProxyErrorTypes.bad_request_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) + + if litellm.store_audit_logs is True: asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -4813,7 +4814,7 @@ async def block_key( object_id=hashed_token, action="blocked", updated_values="{}", - before_value=record.model_dump_json(), + before_value=existing_record.model_dump_json(), ) ) ) @@ -4822,24 +4823,9 @@ async def block_key( where={"token": hashed_token}, data={"blocked": True} # type: ignore ) - ## UPDATE KEY CACHE - - ### get cached object ### - key_object = await get_key_object( + ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB + await _delete_cache_key_object( hashed_token=hashed_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - proxy_logging_obj=proxy_logging_obj, - ) - - ### update cached object ### - key_object.blocked = True - - ### store cached object ### - await _cache_key_object( - hashed_token=hashed_token, - user_api_key_obj=key_object, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -4902,18 +4888,19 @@ async def unblock_key( else: hashed_token = data.key - if litellm.store_audit_logs is True: - # make an audit log for key update - record = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + # Check if the key exists before trying to unblock it + existing_record = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} + ) + if existing_record is None: + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) - if record is None: - raise ProxyException( - message=f"Key {data.key} not found", - type=ProxyErrorTypes.bad_request_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) + + if litellm.store_audit_logs is True: asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -4925,9 +4912,9 @@ async def unblock_key( changed_by_api_key=user_api_key_dict.api_key, table_name=LitellmTableNames.KEY_TABLE_NAME, object_id=hashed_token, - action="blocked", + action="unblocked", updated_values="{}", - before_value=record.model_dump_json(), + before_value=existing_record.model_dump_json(), ) ) ) @@ -4936,24 +4923,9 @@ async def unblock_key( where={"token": hashed_token}, data={"blocked": False} # type: ignore ) - ## UPDATE KEY CACHE - - ### get cached object ### - key_object = await get_key_object( + ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB + await _delete_cache_key_object( hashed_token=hashed_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - proxy_logging_obj=proxy_logging_obj, - ) - - ### update cached object ### - key_object.blocked = False - - ### store cached object ### - await _cache_key_object( - hashed_token=hashed_token, - user_api_key_obj=key_object, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) From b428cfb4a4b249768ef29c9ba6cf1c1134cef52e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:44:22 +0000 Subject: [PATCH 03/57] test: add unit tests for block_key/unblock_key with non-existent keys - test_block_key_nonexistent_key_returns_404: verifies block_key returns 404 (not misleading 401) when the key doesn't exist in the DB - test_unblock_key_nonexistent_key_returns_404: same for unblock_key - test_block_key_existing_key_succeeds: verifies block_key succeeds and invalidates cache for existing keys - Update test_unblock_key_supports_both_sk_and_hashed_tokens to reflect the new cache invalidation pattern (_delete_cache_key_object instead of get_key_object + _cache_key_object) Co-authored-by: yuneng-jiang --- .../test_key_management_endpoints.py | 208 +++++++++++++++++- 1 file changed, 196 insertions(+), 12 deletions(-) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index cfc16808af..49be6ce6bb 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1338,19 +1338,12 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) # Disable audit logs for simpler test # Mock get_key_object and _cache_key_object - async def mock_get_key_object(**kwargs): - return mock_key_object - - async def mock_cache_key_object(**kwargs): + async def mock_delete_cache_key_object(**kwargs): pass monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.get_key_object", - mock_get_key_object, - ) - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints._cache_key_object", - mock_cache_key_object, + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + mock_delete_cache_key_object, ) # Create mock request and user auth @@ -1375,11 +1368,9 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) assert result == mock_key_record - assert mock_key_object.blocked == False # Should be updated to unblocked # Reset mocks for second test mock_prisma_client.db.litellm_verificationtoken.update.reset_mock() - mock_key_object.blocked = True # Reset to blocked state # Test Case 2: Using already hashed token hashed_token_request = BlockKeyRequest(key=test_hashed_token) @@ -1435,6 +1426,199 @@ async def test_unblock_key_invalid_key_format(monkeypatch): assert "Invalid key format" in str(exc_info.value.message) +@pytest.mark.asyncio +async def test_block_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that block_key returns 404 (not misleading 401) when the key + doesn't exist in the database, even when the caller is authenticated + as a proxy admin. + + Previously, block_key would call get_key_object() for cache refresh, + which raised a 401 ProxyException with 'Authentication Error' — making + it look like an auth failure when it was really a missing-key error. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import block_key + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + def mock_hash_token(token): + return "abcd1234" * 8 # 64-char hex + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await block_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + # Must NOT contain "Authentication Error" + assert "Authentication Error" not in str(exc_info.value.message) + # update should never be called since the key doesn't exist + mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_unblock_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that unblock_key returns 404 (not misleading 401) when the key + doesn't exist in the database. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import ( + unblock_key, + ) + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + def mock_hash_token(token): + return "abcd1234" * 8 + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await unblock_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + assert "Authentication Error" not in str(exc_info.value.message) + mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() + + +@pytest.mark.asyncio +async def test_block_key_existing_key_succeeds(monkeypatch): + """ + Test that block_key successfully blocks an existing key and + invalidates the cache entry. + """ + from litellm.proxy._types import BlockKeyRequest + from litellm.proxy.management_endpoints.key_management_endpoints import block_key + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + test_hashed_token = "a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd" + + mock_key_record = MagicMock() + mock_key_record.token = test_hashed_token + mock_key_record.blocked = False + mock_key_record.model_dump_json.return_value = ( + f'{{"token": "{test_hashed_token}", "blocked": false}}' + ) + + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key_record + ) + mock_updated_record = MagicMock() + mock_updated_record.token = test_hashed_token + mock_updated_record.blocked = True + mock_prisma_client.db.litellm_verificationtoken.update = AsyncMock( + return_value=mock_updated_record + ) + + def mock_hash_token(token): + if token.startswith("sk-"): + return test_hashed_token + return token + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.hash_token", mock_hash_token) + monkeypatch.setattr("litellm.store_audit_logs", False) + + # Mock _delete_cache_key_object + async def mock_delete_cache_key_object(**kwargs): + pass + + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object", + mock_delete_cache_key_object, + ) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = BlockKeyRequest(key="sk-test123456789") + + result = await block_key( + data=data, + http_request=mock_request, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify the key was found and updated + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": test_hashed_token} + ) + mock_prisma_client.db.litellm_verificationtoken.update.assert_called_once_with( + where={"token": test_hashed_token}, data={"blocked": True} + ) + assert result == mock_updated_record + + @pytest.mark.asyncio async def test_validate_key_team_change_with_member_permissions(): """ From 5e7645a99b6c432cdafd7cc9a21be3851bd327c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:47:44 +0000 Subject: [PATCH 04/57] chore: remove unused imports (get_key_object, _cache_key_object) These were only used in block_key/unblock_key for cache refresh, which now uses _delete_cache_key_object instead. Co-authored-by: yuneng-jiang --- .../proxy/management_endpoints/key_management_endpoints.py | 2 -- .../management_endpoints/test_key_management_endpoints.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 6dd0d4137b..55def0008b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -41,10 +41,8 @@ from litellm.proxy._experimental.mcp_server.db import ( from litellm.proxy._types import * from litellm.proxy._types import LiteLLM_VerificationToken from litellm.proxy.auth.auth_checks import ( - _cache_key_object, _delete_cache_key_object, can_team_access_model, - get_key_object, get_org_object, get_project_object, get_team_object, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 49be6ce6bb..664a08989e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1314,10 +1314,6 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): return_value=mock_key_record ) - # Mock get_key_object and _cache_key_object functions - mock_key_object = MagicMock() - mock_key_object.blocked = True # Initially blocked - # Mock hash_token function def mock_hash_token(token): if token == "sk-test123456789": @@ -1388,7 +1384,6 @@ async def test_unblock_key_supports_both_sk_and_hashed_tokens(monkeypatch): ) assert result == mock_key_record - assert mock_key_object.blocked == False # Should be updated to unblocked @pytest.mark.asyncio From 3f7f23cd3c0a26203b7de2fdf8b9a17640dfb79b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 18 Mar 2026 08:51:12 +0000 Subject: [PATCH 05/57] chore: restore original dev_config.yaml Co-authored-by: yuneng-jiang --- dev_config.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dev_config.yaml b/dev_config.yaml index 142e0bf94e..64e3c14703 100644 --- a/dev_config.yaml +++ b/dev_config.yaml @@ -1,8 +1,13 @@ model_list: - - model_name: gpt-4 + - model_name: fake-openai-endpoint litellm_params: - model: gpt-4 + model: openai/fake-model api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ general_settings: master_key: sk-1234 + +litellm_settings: + drop_params: True + telemetry: False From 92b89353ae6017bcc2cec821c4a9c6a71c6e0da5 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Mon, 16 Mar 2026 23:22:00 +0100 Subject: [PATCH 06/57] fix: surface Anthropic code execution results as code_interpreter_call in Responses API PR #18945 added support for capturing Anthropic server-side tool results (bash_code_execution_tool_result, etc.) in provider_specific_fields, but the data never reached the Responses API output because: 1. Non-streaming: provider_specific_fields wasn't copied into _hidden_params 2. Streaming: chunk delta's provider_specific_fields wasn't accumulated 3. Tool results weren't mapped to standard output items This fix: - Copies provider_specific_fields to _hidden_params in transform_response() - Accumulates provider_specific_fields from streaming chunk deltas - Maps bash_code_execution_tool_result to code_interpreter_call output items with code and outputs (matching OpenAI's native shape) - Removes redundant function_call items for server-side tools - Adds OutputCodeInterpreterCall type to the output union --- litellm/llms/anthropic/chat/handler.py | 86 +- litellm/llms/anthropic/chat/transformation.py | 79 +- .../streaming_iterator.py | 52 +- .../transformation.py | 57 +- litellm/types/llms/openai.py | 55 +- litellm/types/responses/main.py | 18 + .../chat/test_anthropic_chat_handler.py | 247 ++++- .../test_anthropic_chat_transformation.py | 889 +++++++++--------- 8 files changed, 971 insertions(+), 512 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5eebebc2e2..51b9c9835a 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -48,6 +48,10 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) from litellm.types.utils import ( Delta, GenericStreamingChunk, @@ -538,6 +542,11 @@ class ModelResponseIterator: # Accumulate compaction blocks for multi-turn reconstruction self.compaction_blocks: List[Dict[str, Any]] = [] + # Track server tool use inputs and results for code_interpreter_results + self._server_tool_inputs: Dict[str, Any] = {} + self.tool_results: List[Dict[str, Any]] = [] + self._last_code_interpreter_results_count: int = 0 + def check_empty_tool_call_args(self) -> bool: """ Check if the tool call block so far has been an empty string @@ -568,9 +577,7 @@ class ModelResponseIterator: speed=self.speed, ) - def _content_block_delta_helper( - self, chunk: dict - ) -> Tuple[ + def _content_block_delta_helper(self, chunk: dict) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], @@ -682,6 +689,44 @@ class ModelResponseIterator: return content_block_start + def _build_code_interpreter_results(self) -> list: + """Convert accumulated tool_results to OutputCodeInterpreterCall objects. + + Called during streaming to produce provider-neutral code_interpreter_results + alongside the raw tool_results, so the Responses API layer doesn't need + Anthropic-specific knowledge. + """ + # Only convert tool_results added since the last call to avoid + # duplicates when _merge_provider_specific_fields extends the list. + new_results = self.tool_results[self._last_code_interpreter_results_count :] + self._last_code_interpreter_results_count = len(self.tool_results) + results = [] + for tr in new_results: + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + if isinstance(content, dict): + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) if parts else str(content) + else: + logs = str(content) + tool_input = self._server_tool_inputs.get(call_id, {}) + code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" + results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code, + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs=logs)], + ) + ) + return results + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915 try: type_chunk = chunk.get("type", "") or "" @@ -748,6 +793,17 @@ class ModelResponseIterator: ), index=self.tool_index, ) + # Track server tool use inputs for code_interpreter_results + if ( + content_block_start["content_block"]["type"] + == "server_tool_use" + ): + tool_input = content_block_start["content_block"].get( + "input", {} + ) + self._server_tool_inputs[ + content_block_start["content_block"]["id"] + ] = tool_input # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -768,9 +824,9 @@ class ModelResponseIterator: # Handle compaction blocks # The full content comes in content_block_start self.compaction_blocks.append(content_block_start["content_block"]) - provider_specific_fields[ - "compaction_blocks" - ] = self.compaction_blocks + provider_specific_fields["compaction_blocks"] = ( + self.compaction_blocks + ) provider_specific_fields["compaction_start"] = { "type": "compaction", "content": content_block_start["content_block"].get( @@ -792,9 +848,9 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields[ - "web_search_results" - ] = self.web_search_results + provider_specific_fields["web_search_results"] = ( + self.web_search_results + ) elif content_type == "web_fetch_tool_result": # Capture web_fetch_tool_result for multi-turn reconstruction # The full content comes in content_block_start, not in deltas @@ -802,16 +858,18 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields[ - "web_search_results" - ] = self.web_search_results + provider_specific_fields["web_search_results"] = ( + self.web_search_results + ) elif content_type != "tool_search_tool_result": # Handle other tool results (code execution, etc.) # Skip tool_search_tool_result as it's internal metadata - if not hasattr(self, "tool_results"): - self.tool_results = [] self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results + # Convert to provider-neutral code_interpreter_results + provider_specific_fields["code_interpreter_results"] = ( + self._build_code_interpreter_results() + ) elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 47cdd8287e..033afea2ff 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -59,6 +59,10 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, ServerToolUse, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) from litellm.utils import ( ModelResponse, Usage, @@ -960,11 +964,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if mcp_servers: optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": - _tool_choice: Optional[ - AnthropicMessagesToolChoice - ] = self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), + _tool_choice: Optional[AnthropicMessagesToolChoice] = ( + self._map_tool_choice( + tool_choice=non_default_params.get("tool_choice"), + parallel_tool_use=non_default_params.get("parallel_tool_calls"), + ) ) if _tool_choice is not None: @@ -1062,9 +1066,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.map_openai_context_management_to_anthropic(value) ) if anthropic_context_management is not None: - optional_params[ - "context_management" - ] = anthropic_context_management + optional_params["context_management"] = ( + anthropic_context_management + ) elif param == "speed" and isinstance(value, str): # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value @@ -1138,9 +1142,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text=system_message_block["content"], ) if "cache_control" in system_message_block: - anthropic_system_message_content[ - "cache_control" - ] = system_message_block["cache_control"] + anthropic_system_message_content["cache_control"] = ( + system_message_block["cache_control"] + ) anthropic_system_message_list.append( anthropic_system_message_content ) @@ -1164,9 +1168,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) if "cache_control" in _content: - anthropic_system_message_content[ - "cache_control" - ] = _content["cache_control"] + anthropic_system_message_content["cache_control"] = ( + _content["cache_control"] + ) anthropic_system_message_list.append( anthropic_system_message_content @@ -1463,9 +1467,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _message - def extract_response_content( - self, completion_response: dict - ) -> Tuple[ + def extract_response_content(self, completion_response: dict) -> Tuple[ str, Optional[List[Any]], Optional[ @@ -1749,6 +1751,48 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): provider_specific_fields["web_search_results"] = web_search_results if tool_results is not None: provider_specific_fields["tool_results"] = tool_results + # Convert to provider-neutral OutputCodeInterpreterCall objects + # so the Responses API layer can use them without Anthropic-specific knowledge. + container_id = ( + completion_response.get("container", {}).get("id") + if isinstance(completion_response.get("container"), dict) + else None + ) + code_by_id: Dict[str, str] = {} + for tc in tool_calls: + try: + args = json.loads(tc.get("function", {}).get("arguments", "{}")) + code_by_id[tc.get("id", "")] = args.get("command", "") + except Exception: + pass + code_interpreter_results = [] + for tr in tool_results: + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + if isinstance(content, dict): + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) if parts else str(content) + else: + logs = str(content) + code_interpreter_results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code_by_id.get(call_id, ""), + container_id=container_id, + status="completed", + outputs=[ + OutputCodeInterpreterCallLog(type="logs", logs=logs) + ], + ) + ) + provider_specific_fields["code_interpreter_results"] = ( + code_interpreter_results + ) if container is not None: provider_specific_fields["container"] = container if compaction_blocks is not None: @@ -1794,6 +1838,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response.created = int(time.time()) model_response.model = completion_response["model"] + _hidden_params["provider_specific_fields"] = provider_specific_fields model_response._hidden_params = _hidden_params return model_response diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index ce037850b8..0b7d6e8a7a 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -107,6 +107,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._reasoning_done_emitted = False self._reasoning_item_id: Optional[str] = None self._accumulated_reasoning_content_parts: List[str] = [] + self._accumulated_provider_specific_fields: Dict[str, Any] = {} def _get_or_assign_tool_output_index(self, call_id: str) -> int: existing = self._tool_output_index_by_call_id.get(call_id) @@ -479,16 +480,37 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): event.__dict__["sequence_number"] = self._sequence_number return event - def create_litellm_model_response( - self, - ) -> Optional[ModelResponse]: - return cast( + def _merge_provider_specific_fields(self, src: dict) -> None: + """Merge provider_specific_fields, extending list values instead of replacing.""" + for key, val in src.items(): + existing = self._accumulated_provider_specific_fields.get(key) + if ( + existing is not None + and isinstance(val, list) + and isinstance(existing, list) + ): + existing.extend(val) + else: + self._accumulated_provider_specific_fields[key] = val + + def create_litellm_model_response(self) -> Optional[ModelResponse]: + response = cast( Optional[ModelResponse], stream_chunk_builder( chunks=self.collected_chat_completion_chunks, logging_obj=self.litellm_logging_obj, ), ) + if response is not None and self._accumulated_provider_specific_fields: + if ( + not hasattr(response, "_hidden_params") + or response._hidden_params is None + ): + response._hidden_params = {} + response._hidden_params.setdefault("provider_specific_fields", {}).update( + self._accumulated_provider_specific_fields + ) + return response @staticmethod def _snapshot_chunk_for_stream_chunk_builder( @@ -853,6 +875,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if chunk is not None: chunk = cast(ModelResponseStream, chunk) self._ensure_output_item_for_chunk(chunk) + # Accumulate provider_specific_fields from chunk and delta + for src in ( + getattr(chunk, "provider_specific_fields", None), + getattr( + chunk.choices[0].delta if chunk.choices else None, + "provider_specific_fields", + None, + ), + ): + if src and isinstance(src, dict): + self._merge_provider_specific_fields(src) # Proceed to transformation self.collected_chat_completion_chunks.append( self._snapshot_chunk_for_stream_chunk_builder(chunk) @@ -964,6 +997,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: chunk = self.litellm_custom_stream_wrapper.__next__() self._ensure_output_item_for_chunk(chunk) + # Accumulate provider_specific_fields from chunk and delta + for src in ( + getattr(chunk, "provider_specific_fields", None), + getattr( + chunk.choices[0].delta if chunk.choices else None, + "provider_specific_fields", + None, + ), + ): + if src and isinstance(src, dict): + self._merge_provider_specific_fields(src) # Emit any just-queued output_item event if self._pending_response_events: return self._pending_response_events.pop(0) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 71fa88fb75..b54d5930ef 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -42,6 +42,7 @@ from litellm.types.llms.openai import ( from litellm.types.responses.main import ( GenericResponseOutputItem, GenericResponseOutputItemContentAnnotation, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, OutputText, @@ -1696,6 +1697,7 @@ class LiteLLMCompletionResponsesConfig: ) -> List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1704,6 +1706,7 @@ class LiteLLMCompletionResponsesConfig: responses_output: List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1725,8 +1728,56 @@ class LiteLLMCompletionResponsesConfig: chat_completion_response=chat_completion_response ) ) + + # Convert server-side tool results (e.g. Anthropic code execution) + # into code_interpreter_call output items, replacing the corresponding + # function_call items so the output matches OpenAI's native shape. + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items( + chat_completion_response + ) + ) + if tool_result_items: + result_by_id = {item.id: item for item in tool_result_items} + replaced_ids = set(result_by_id.keys()) + responses_output = [ + ( + result_by_id[getattr(item, "call_id", None)] + if ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in replaced_ids + ) + else item + ) + for item in responses_output + ] + return responses_output + @staticmethod + def _extract_tool_result_output_items( + chat_completion_response: ModelResponse, + ) -> list: + """Extract pre-built code_interpreter_call output items from provider_specific_fields. + + Provider transformers (e.g. Anthropic) convert their native tool results + into OutputCodeInterpreterCall objects and store them in + provider_specific_fields["code_interpreter_results"]. This method + simply retrieves them — no provider-specific parsing here. + """ + output_items: list = [] + for choice in chat_completion_response.choices or []: + message = getattr(choice, "message", None) + if not message: + continue + psf = getattr(message, "provider_specific_fields", None) + if not psf or not isinstance(psf, dict): + continue + results = psf.get("code_interpreter_results") + if results and isinstance(results, list): + output_items.extend(results) + return output_items + @staticmethod def _extract_reasoning_output_items( chat_completion_response: ModelResponse, @@ -2055,9 +2106,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict[ - "reasoning_tokens" - ] = completion_details.reasoning_tokens + output_details_dict["reasoning_tokens"] = ( + completion_details.reasoning_tokens + ) else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a2df3f2e0d..a265198e6b 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -84,6 +84,7 @@ from typing_extensions import Annotated, Dict, Required, TypedDict, override from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from litellm.types.responses.main import ( GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ) @@ -969,12 +970,12 @@ class OpenAIChatCompletionChunk(ChatCompletionChunk): class Hyperparameters(BaseModel): batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." - learning_rate_multiplier: Optional[ - Union[str, float] - ] = None # Scaling factor for the learning rate - n_epochs: Optional[ - Union[str, int] - ] = None # "The number of epochs to train the model for" + learning_rate_multiplier: Optional[Union[str, float]] = ( + None # Scaling factor for the learning rate + ) + n_epochs: Optional[Union[str, int]] = ( + None # "The number of epochs to train the model for" + ) model_config = {"extra": "allow"} @@ -1003,18 +1004,18 @@ class FineTuningJobCreate(BaseModel): model: str # "The name of the model to fine-tune." training_file: str # "The ID of an uploaded file that contains training data." - hyperparameters: Optional[ - Hyperparameters - ] = None # "The hyperparameters used for the fine-tuning job." - suffix: Optional[ - str - ] = None # "A string of up to 18 characters that will be added to your fine-tuned model name." - validation_file: Optional[ - str - ] = None # "The ID of an uploaded file that contains validation data." - integrations: Optional[ - List[str] - ] = None # "A list of integrations to enable for your fine-tuning job." + hyperparameters: Optional[Hyperparameters] = ( + None # "The hyperparameters used for the fine-tuning job." + ) + suffix: Optional[str] = ( + None # "A string of up to 18 characters that will be added to your fine-tuned model name." + ) + validation_file: Optional[str] = ( + None # "The ID of an uploaded file that contains validation data." + ) + integrations: Optional[List[str]] = ( + None # "A list of integrations to enable for your fine-tuning job." + ) seed: Optional[int] = None # "The seed controls the reproducibility of the job." @@ -1242,6 +1243,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): List[ Union[ GenericResponseOutputItem, + OutputCodeInterpreterCall, OutputFunctionToolCall, OutputImageGenerationCall, ResponseFunctionToolCall, @@ -1308,13 +1310,16 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): if not isinstance(serialized, list): return serialized return [ - { - k: v - for k, v in item.items() - if v is not None or k not in ("status", "content", "encrypted_content") - } - if isinstance(item, dict) and item.get("type") == "reasoning" - else item + ( + { + k: v + for k, v in item.items() + if v is not None + or k not in ("status", "content", "encrypted_content") + } + if isinstance(item, dict) and item.get("type") == "reasoning" + else item + ) for item in serialized ] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 7a666d5e65..e46857565c 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -49,6 +49,24 @@ class OutputImageGenerationCall(BaseLiteLLMOpenAIResponseObject): result: Optional[str] # Base64 encoded image data (without data:image prefix) +class OutputCodeInterpreterCallLog(BaseLiteLLMOpenAIResponseObject): + """Log output from a code interpreter call""" + + type: Literal["logs"] + logs: str + + +class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): + """A code interpreter / code execution call output""" + + type: Literal["code_interpreter_call"] + id: str + code: Optional[str] + container_id: Optional[str] + status: Literal["in_progress", "completed", "incomplete", "failed"] + outputs: Optional[List[OutputCodeInterpreterCallLog]] + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index d9f513d8d1..35c7a62027 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -6,6 +6,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ) +from litellm.types.responses.main import OutputCodeInterpreterCall def test_redacted_thinking_content_block_delta(): @@ -479,14 +480,22 @@ def test_partial_json_chunk_accumulation(): # First partial chunk should return None (still accumulating) result1 = iterator._parse_sse_data(f"data:{partial_chunk_1}") assert result1 is None, "First partial chunk should return None while accumulating" - assert iterator.chunk_type == "accumulated_json", "Should switch to accumulated_json mode" - assert iterator.accumulated_json == partial_chunk_1, "Should have accumulated first part" + assert ( + iterator.chunk_type == "accumulated_json" + ), "Should switch to accumulated_json mode" + assert ( + iterator.accumulated_json == partial_chunk_1 + ), "Should have accumulated first part" # Second partial chunk should complete the JSON and return a parsed result result2 = iterator._parse_sse_data(f"data:{partial_chunk_2}") assert result2 is not None, "Second chunk should return parsed result" - assert iterator.accumulated_json == "", "Buffer should be cleared after successful parse" - assert result2.choices[0].delta.content == "Hello", f"Expected 'Hello', got '{result2.choices[0].delta.content}'" + assert ( + iterator.accumulated_json == "" + ), "Buffer should be cleared after successful parse" + assert ( + result2.choices[0].delta.content == "Hello" + ), f"Expected 'Hello', got '{result2.choices[0].delta.content}'" def test_complete_json_chunk_no_accumulation(): @@ -503,7 +512,9 @@ def test_complete_json_chunk_no_accumulation(): assert result is not None, "Complete chunk should return parsed result immediately" assert iterator.chunk_type == "valid_json", "Should remain in valid_json mode" assert iterator.accumulated_json == "", "Buffer should remain empty" - assert result.choices[0].delta.content == "Hello", f"Expected 'Hello', got '{result.choices[0].delta.content}'" + assert ( + result.choices[0].delta.content == "Hello" + ), f"Expected 'Hello', got '{result.choices[0].delta.content}'" def test_multiple_partial_chunks_accumulation(): @@ -620,7 +631,9 @@ def test_web_search_tool_result_no_extra_tool_calls(): # Should have exactly 2 tool calls: # 1. From content_block_start (server_tool_use) with id and name # 2. From content_block_delta with the actual query - assert len(tool_calls_emitted) == 2, f"Expected 2 tool calls, got {len(tool_calls_emitted)}" + assert ( + len(tool_calls_emitted) == 2 + ), f"Expected 2 tool calls, got {len(tool_calls_emitted)}" # First tool call should have the id and name assert tool_calls_emitted[0]["id"] == "srvtoolu_01ABC123" @@ -722,7 +735,10 @@ def test_web_search_tool_result_captured_in_provider_specific_fields(): { "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"query": "otter facts"}'}, + "delta": { + "type": "input_json_delta", + "partial_json": '{"query": "otter facts"}', + }, }, # 4. content_block_stop for server_tool_use {"type": "content_block_stop", "index": 0}, @@ -822,7 +838,10 @@ def test_web_fetch_tool_result_captured_in_provider_specific_fields(): { "type": "content_block_delta", "index": 0, - "delta": {"type": "input_json_delta", "partial_json": '{"url": "https://example.com"}'}, + "delta": { + "type": "input_json_delta", + "partial_json": '{"url": "https://example.com"}', + }, }, # 4. content_block_stop for server_tool_use {"type": "content_block_stop", "index": 0}, @@ -946,7 +965,7 @@ def test_web_fetch_tool_result_no_extra_tool_calls(): def test_container_in_provider_specific_fields_streaming(): """ Test that container is captured in provider_specific_fields for streaming responses. - + When container with skills is used, the container field should be present in the provider_specific_fields of the message_delta chunk. """ @@ -1025,7 +1044,9 @@ def test_container_in_provider_specific_fields_streaming(): ] # Verify container was captured - assert container_field is not None, "container should be captured in provider_specific_fields" + assert ( + container_field is not None + ), "container should be captured in provider_specific_fields" assert ( container_field["id"] == "container_011CW9hA9zpZ8xD3bjjShy4p" ), "container id should match" @@ -1033,18 +1054,14 @@ def test_container_in_provider_specific_fields_streaming(): container_field["expires_at"] == "2025-12-16T04:57:16.913181Z" ), "expires_at should match" assert len(container_field["skills"]) == 1, "Should have 1 skill" - assert ( - container_field["skills"][0]["skill_id"] == "pptx" - ), "skill_id should be pptx" - assert ( - container_field["skills"][0]["version"] == "20251013" - ), "version should match" + assert container_field["skills"][0]["skill_id"] == "pptx", "skill_id should be pptx" + assert container_field["skills"][0]["version"] == "20251013", "version should match" def test_container_in_provider_specific_fields_non_streaming(): """ Test that container is captured in provider_specific_fields for non-streaming responses. - + When container with skills is used in non-streaming, the container field should be present in the provider_specific_fields of the response. """ @@ -1106,7 +1123,7 @@ def test_container_in_provider_specific_fields_non_streaming(): def test_container_absent_when_not_provided(): """ Test that container is not added to provider_specific_fields when not provided. - + This ensures we don't add empty or None container fields. """ iterator = ModelResponseIterator( @@ -1133,3 +1150,197 @@ def test_container_absent_when_not_provided(): assert ( "container" not in model_response.choices[0].delta.provider_specific_fields ), "container should not be present when not provided in delta" + + +def test_streaming_code_execution_produces_code_interpreter_results(): + """ + Test that bash_code_execution_tool_result content blocks in streaming + produce code_interpreter_results in provider_specific_fields, so the + Responses API layer can use them without Anthropic-specific knowledge. + """ + + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "text", + "text": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Running code..."}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01ABC", + "name": "bash_code_execution", + "input": {"command": "echo hello"}, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "content_block_start", + "index": 2, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01ABC", + "content": { + "type": "bash_code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 2}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + found_code_interpreter_results = False + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + found_code_interpreter_results = True + results = psf["code_interpreter_results"] + assert len(results) == 1 + assert isinstance(results[0], OutputCodeInterpreterCall) + assert results[0].type == "code_interpreter_call" + assert results[0].id == "srvtoolu_01ABC" + assert results[0].code == "echo hello" + assert results[0].outputs is not None + assert len(results[0].outputs) == 1 + assert results[0].outputs[0].logs == "hello\n" + + assert found_code_interpreter_results, ( + "code_interpreter_results should appear in provider_specific_fields " + "when bash_code_execution_tool_result is streamed" + ) + + +def test_streaming_multiple_code_executions_no_duplicates(): + """ + Test that multiple code executions in a single streaming response produce + exactly one code_interpreter_result per execution — no duplicates from + _build_code_interpreter_results rebuilding the full list. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + # First code execution + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {"command": "echo first"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "first\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + # Second code execution + { + "type": "content_block_start", + "index": 2, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01BBB", + "name": "bash_code_execution", + "input": {"command": "echo second"}, + }, + }, + {"type": "content_block_stop", "index": 2}, + { + "type": "content_block_start", + "index": 3, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BBB", + "content": { + "type": "bash_code_execution_result", + "stdout": "second\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 3}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + # Collect ALL code_interpreter_results emitted across all chunks + all_results = [] + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + all_results.extend(psf["code_interpreter_results"]) + + # Should have exactly 2 results, one per execution — no duplicates + assert len(all_results) == 2, ( + f"Expected 2 code_interpreter_results, got {len(all_results)}. " + f"IDs: {[r.id for r in all_results]}" + ) + assert all_results[0].id == "srvtoolu_01AAA" + assert all_results[0].code == "echo first" + assert all_results[0].outputs[0].logs == "first\n" + assert all_results[1].id == "srvtoolu_01BBB" + assert all_results[1].code == "echo second" + assert all_results[1].outputs[0].logs == "second\n" diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a95b9413b9..a346996486 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -183,7 +183,9 @@ def test_extract_response_content_with_citations(): }, } - _, citations, _, _, _, _, _, _ = config.extract_response_content(completion_response) + _, citations, _, _, _, _, _, _ = config.extract_response_content( + completion_response + ) assert citations == [ [ { @@ -305,7 +307,7 @@ def test_web_search_tool_result_extraction(): "type": "server_tool_use", "id": "srvtoolu_01ABC123", "name": "web_search", - "input": {"query": "average weight african elephant kg"} + "input": {"query": "average weight african elephant kg"}, }, { "type": "web_search_tool_result", @@ -317,32 +319,39 @@ def test_web_search_tool_result_extraction(): "title": "African Elephant Facts", "encrypted_content": "encrypted_data_here", "page_age": "2024-01-15", - "snippet": "Adult African elephants weigh between 4,000-6,000 kg..." + "snippet": "Adult African elephants weigh between 4,000-6,000 kg...", } - ] + ], }, { "type": "text", - "text": "Based on my search, African elephants weigh around 5,000 kg." + "text": "Based on my search, African elephants weigh around 5,000 kg.", }, { "type": "tool_use", "id": "toolu_01XYZ789", "name": "add_numbers", - "input": {"a": 5000, "b": 100} - } + "input": {"a": 5000, "b": 100}, + }, ], "stop_reason": "tool_use", "usage": { "input_tokens": 100, "output_tokens": 50, - "server_tool_use": {"web_search_requests": 1} - } + "server_tool_use": {"web_search_requests": 1}, + }, } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify text extraction assert "Based on my search" in text @@ -388,7 +397,7 @@ def test_web_search_tool_result_in_provider_specific_fields(): "type": "server_tool_use", "id": "srvtoolu_provider_test", "name": "web_search", - "input": {"query": "test query"} + "input": {"query": "test query"}, }, { "type": "web_search_tool_result", @@ -398,21 +407,18 @@ def test_web_search_tool_result_in_provider_specific_fields(): "type": "web_search_result", "url": "https://example.com/test", "title": "Test Result", - "snippet": "Test snippet content" + "snippet": "Test snippet content", } - ] + ], }, - { - "type": "text", - "text": "Here is the result." - } + {"type": "text", "text": "Here is the result."}, ], "stop_reason": "end_turn", "usage": { "input_tokens": 50, "output_tokens": 25, - "server_tool_use": {"web_search_requests": 1} - } + "server_tool_use": {"web_search_requests": 1}, + }, } raw_response = httpx.Response(status_code=200, headers={}) @@ -432,7 +438,10 @@ def test_web_search_tool_result_in_provider_specific_fields(): assert "web_search_results" in provider_fields assert len(provider_fields["web_search_results"]) == 1 assert provider_fields["web_search_results"][0]["type"] == "web_search_tool_result" - assert provider_fields["web_search_results"][0]["tool_use_id"] == "srvtoolu_provider_test" + assert ( + provider_fields["web_search_results"][0]["tool_use_id"] + == "srvtoolu_provider_test" + ) def test_multiple_web_search_tool_results(): @@ -447,34 +456,52 @@ def test_multiple_web_search_tool_results(): "type": "server_tool_use", "id": "srvtoolu_search1", "name": "web_search", - "input": {"query": "african elephant weight"} + "input": {"query": "african elephant weight"}, }, { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_search1", - "content": [{"type": "web_search_result", "url": "https://example1.com", "title": "Result 1", "snippet": "First result"}] + "content": [ + { + "type": "web_search_result", + "url": "https://example1.com", + "title": "Result 1", + "snippet": "First result", + } + ], }, { "type": "server_tool_use", "id": "srvtoolu_search2", "name": "web_search", - "input": {"query": "asian elephant weight"} + "input": {"query": "asian elephant weight"}, }, { "type": "web_search_tool_result", "tool_use_id": "srvtoolu_search2", - "content": [{"type": "web_search_result", "url": "https://example2.com", "title": "Result 2", "snippet": "Second result"}] + "content": [ + { + "type": "web_search_result", + "url": "https://example2.com", + "title": "Result 2", + "snippet": "Second result", + } + ], }, - { - "type": "text", - "text": "Found information about both elephants." - } + {"type": "text", "text": "Found information about both elephants."}, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify both web_search_tool_results are extracted assert web_search_results is not None @@ -751,7 +778,7 @@ def test_anthropic_beta_header_merging_with_output_format(): optional_params = { "output_format": { "type": "json_schema", - "schema": {"type": "object", "properties": {}} + "schema": {"type": "object", "properties": {}}, } } @@ -761,10 +788,12 @@ def test_anthropic_beta_header_merging_with_output_format(): # Both beta headers should be present beta_value = result_headers["anthropic-beta"] - assert "context-1m-2025-08-07" in beta_value, \ - f"User's context-1m beta header missing from: {beta_value}" - assert "structured-outputs-2025-11-13" in beta_value, \ - f"Structured output beta header missing from: {beta_value}" + assert ( + "context-1m-2025-08-07" in beta_value + ), f"User's context-1m beta header missing from: {beta_value}" + assert ( + "structured-outputs-2025-11-13" in beta_value + ), f"Structured output beta header missing from: {beta_value}" def test_anthropic_beta_header_merging_with_multiple_features(): @@ -780,10 +809,10 @@ def test_anthropic_beta_header_merging_with_multiple_features(): optional_params = { "output_format": { "type": "json_schema", - "schema": {"type": "object", "properties": {}} + "schema": {"type": "object", "properties": {}}, }, "context_management": _sample_context_management_payload(), - "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}] + "tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}], } result_headers = config.update_headers_with_optional_anthropic_beta( @@ -950,20 +979,12 @@ def test_tool_search_regex_detection(): # Test with tool search regex tool tools = [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - } + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} ] assert config.is_tool_search_used(tools) is True # Test without tool search - tools = [ - { - "type": "function", - "function": {"name": "get_weather"} - } - ] + tools = [{"type": "function", "function": {"name": "get_weather"}}] assert config.is_tool_search_used(tools) is False @@ -975,10 +996,7 @@ def test_tool_search_bm25_detection(): # Test with tool search BM25 tool tools = [ - { - "type": "tool_search_tool_bm25_20251119", - "name": "tool_search_tool_bm25" - } + {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} ] assert config.is_tool_search_used(tools) is True @@ -1002,10 +1020,7 @@ def test_tool_search_regex_mapping(): """Test that tool search regex tools are properly mapped""" config = AnthropicConfig() - tool = { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - } + tool = {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"} mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1019,10 +1034,7 @@ def test_tool_search_bm25_mapping(): """Test that tool search BM25 tools are properly mapped""" config = AnthropicConfig() - tool = { - "type": "tool_search_tool_bm25_20251119", - "name": "tool_search_tool_bm25" - } + tool = {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"} mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1037,20 +1049,17 @@ def test_deferred_tools_separation(): config = AnthropicConfig() tools = [ - { - "type": "tool_search_tool_regex_20251119", - "name": "tool_search_tool_regex" - }, + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, { "type": "function", "function": {"name": "get_weather"}, - "defer_loading": True + "defer_loading": True, }, { "type": "function", "function": {"name": "search_files"}, - "defer_loading": False - } + "defer_loading": False, + }, ] non_deferred, deferred = config._separate_deferred_tools(tools) @@ -1069,14 +1078,21 @@ def test_server_tool_use_in_response(): "type": "server_tool_use", "id": "srvtoolu_01ABC123", "name": "tool_search_tool_regex", - "input": {"query": "weather"} + "input": {"query": "weather"}, } ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) assert len(tool_calls) == 1 assert tool_calls[0]["id"] == "srvtoolu_01ABC123" @@ -1091,9 +1107,7 @@ def test_tool_search_usage_tracking(): usage_object = { "input_tokens": 100, "output_tokens": 50, - "server_tool_use": { - "tool_search_requests": 2 - } + "server_tool_use": {"tool_search_requests": 2}, } usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) @@ -1109,16 +1123,13 @@ def test_tool_reference_expansion(): deferred_tools = [ { "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather" - } + "function": {"name": "get_weather", "description": "Get weather"}, } ] content = [ {"type": "text", "text": "I'll search for tools"}, - {"type": "tool_reference", "tool_name": "get_weather"} + {"type": "tool_reference", "tool_name": "get_weather"}, ] expanded = config._expand_tool_references(content, deferred_tools) @@ -1140,13 +1151,11 @@ def test_defer_loading_preserved_in_transformation(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - "defer_loading": True + "defer_loading": True, } mapped_tool, mcp_server = config._map_tool_helper(tool) @@ -1166,45 +1175,51 @@ def test_tool_search_complete_response_parsing(): "content": [ { "type": "text", - "text": "I'll search for weather-related tools that can help you." + "text": "I'll search for weather-related tools that can help you.", }, { "type": "server_tool_use", "id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "name": "tool_search_tool_regex", "input": {"pattern": "weather", "limit": 5}, - "caller": {"type": "direct"} + "caller": {"type": "direct"}, }, { "type": "tool_search_tool_result", "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", "content": { "type": "tool_search_tool_search_result", - "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}] - } - }, - { - "type": "text", - "text": "Great! I found a weather tool." + "tool_references": [ + {"type": "tool_reference", "tool_name": "get_weather"} + ], + }, }, + {"type": "text", "text": "Great! I found a weather tool."}, { "type": "tool_use", "id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ", "name": "get_weather", - "input": {"location": "San Francisco"} - } + "input": {"location": "San Francisco"}, + }, ], "usage": { "input_tokens": 1639, "output_tokens": 170, - "server_tool_use": {"web_search_requests": 0} - } + "server_tool_use": {"web_search_requests": 0}, + }, } # Extract content - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify text extraction (should concatenate both text blocks) assert "I'll search for weather-related tools" in text @@ -1222,12 +1237,14 @@ def test_tool_search_complete_response_parsing(): usage = config.calculate_usage( usage_object=completion_response["usage"], reasoning_content=None, - completion_response=completion_response + completion_response=completion_response, ) assert usage.server_tool_use is not None assert usage.server_tool_use.web_search_requests == 0 - assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + assert ( + usage.server_tool_use.tool_search_requests == 1 + ) # Counted from server_tool_use blocks def test_allowed_callers_field_preservation(): @@ -1242,13 +1259,11 @@ def test_allowed_callers_field_preservation(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] - } + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], + }, }, - "allowed_callers": ["code_execution_20250825"] + "allowed_callers": ["code_execution_20250825"], } transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers) @@ -1265,19 +1280,16 @@ def test_programmatic_tool_calling_beta_header(): # Test detection with allowed_callers tools = [ - { - "type": "code_execution_20250825", - "name": "code_execution" - }, + {"type": "code_execution_20250825", "name": "code_execution"}, { "type": "function", "function": { "name": "query_database", "description": "Execute a SQL query", - "parameters": {"type": "object", "properties": {}} + "parameters": {"type": "object", "properties": {}}, }, - "allowed_callers": ["code_execution_20250825"] - } + "allowed_callers": ["code_execution_20250825"], + }, ] is_programmatic = model_info.is_programmatic_tool_calling_used(tools) @@ -1285,8 +1297,7 @@ def test_programmatic_tool_calling_beta_header(): # Test header generation headers = model_info.get_anthropic_headers( - api_key="test-key", - programmatic_tool_calling_used=True + api_key="test-key", programmatic_tool_calling_used=True ) assert "anthropic-beta" in headers @@ -1303,10 +1314,7 @@ def test_caller_field_in_response(): "type": "message", "role": "assistant", "content": [ - { - "type": "text", - "text": "I'll query the database." - }, + {"type": "text", "text": "I'll query the database."}, { "type": "tool_use", "id": "toolu_123", @@ -1314,15 +1322,24 @@ def test_caller_field_in_response(): "input": {"sql": "SELECT * FROM users"}, "caller": { "type": "code_execution_20250825", - "tool_id": "srvtoolu_abc" - } - } + "tool_id": "srvtoolu_abc", + }, + }, ], "stop_reason": "tool_use", - "usage": {"input_tokens": 100, "output_tokens": 50} + "usage": {"input_tokens": 100, "output_tokens": 50}, } - text, citations, thinking, reasoning, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content(completion_response) + ( + text, + citations, + thinking, + reasoning, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) assert len(tool_calls) == 1 assert tool_calls[0]["id"] == "toolu_123" @@ -1337,10 +1354,7 @@ def test_code_execution_20250825_tool_type(): """Test that code_execution_20250825 tool type is handled correctly.""" config = AnthropicConfig() - tool = { - "type": "code_execution_20250825", - "name": "code_execution" - } + tool = {"type": "code_execution_20250825", "name": "code_execution"} transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None @@ -1360,13 +1374,11 @@ def test_allowed_callers_in_function_field(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], }, - "allowed_callers": ["code_execution_20250825"] - } + "allowed_callers": ["code_execution_20250825"], + }, } transformed_tool, _ = config._map_tool_helper(tool) @@ -1389,15 +1401,15 @@ def test_input_examples_field_preservation(): "type": "object", "properties": { "location": {"type": "string"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, }, - "required": ["location"] - } + "required": ["location"], + }, }, "input_examples": [ {"location": "San Francisco, CA", "unit": "fahrenheit"}, - {"location": "Tokyo, Japan", "unit": "celsius"} - ] + {"location": "Tokyo, Japan", "unit": "celsius"}, + ], } transformed_tool, _ = config._map_tool_helper(tool_with_examples) @@ -1420,11 +1432,9 @@ def test_input_examples_beta_header(): "function": { "name": "get_weather", "description": "Get weather information", - "parameters": {"type": "object", "properties": {}} + "parameters": {"type": "object", "properties": {}}, }, - "input_examples": [ - {"location": "San Francisco, CA"} - ] + "input_examples": [{"location": "San Francisco, CA"}], } ] @@ -1433,8 +1443,7 @@ def test_input_examples_beta_header(): # Test header generation headers = model_info.get_anthropic_headers( - api_key="test-key", - input_examples_used=True + api_key="test-key", input_examples_used=True ) assert "anthropic-beta" in headers @@ -1453,16 +1462,14 @@ def test_input_examples_in_function_field(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] + "properties": {"location": {"type": "string"}}, + "required": ["location"], }, "input_examples": [ {"location": "Paris, France"}, - {"location": "London, UK"} - ] - } + {"location": "London, UK"}, + ], + }, } transformed_tool, _ = config._map_tool_helper(tool) @@ -1483,17 +1490,13 @@ def test_input_examples_with_other_features(): "description": "Execute a SQL query", "parameters": { "type": "object", - "properties": { - "sql": {"type": "string"} - }, - "required": ["sql"] - } + "properties": {"sql": {"type": "string"}}, + "required": ["sql"], + }, }, - "input_examples": [ - {"sql": "SELECT * FROM users WHERE id = 1"} - ], + "input_examples": [{"sql": "SELECT * FROM users WHERE id = 1"}], "defer_loading": True, - "allowed_callers": ["code_execution_20250825"] + "allowed_callers": ["code_execution_20250825"], } transformed_tool, _ = config._map_tool_helper(tool) @@ -1517,19 +1520,20 @@ def test_input_examples_empty_list_not_added(): "description": "Get weather information", "parameters": { "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, }, - "input_examples": [] + "input_examples": [], } transformed_tool, _ = config._map_tool_helper(tool) assert transformed_tool is not None # Empty list should not be added - assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + assert ( + "input_examples" not in transformed_tool + or len(transformed_tool.get("input_examples", [])) == 0 + ) # ============ Effort Parameter Tests ============ @@ -1540,18 +1544,14 @@ def test_effort_output_config_preservation(): config = AnthropicConfig() messages = [{"role": "user", "content": "Analyze this code"}] - optional_params = { - "output_config": { - "effort": "medium" - } - } + optional_params = {"output_config": {"effort": "medium"}} result = config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert "output_config" in result @@ -1565,18 +1565,13 @@ def test_effort_beta_header_injection(): model_info = AnthropicModelInfo() # Test with effort parameter - optional_params = { - "output_config": { - "effort": "low" - } - } + optional_params = {"output_config": {"effort": "low"}} effort_used = model_info.is_effort_used(optional_params=optional_params) assert effort_used is True headers = model_info.get_anthropic_headers( - api_key="test-key", - effort_used=effort_used + api_key="test-key", effort_used=effort_used ) assert "anthropic-beta" in headers @@ -1597,7 +1592,7 @@ def test_effort_validation(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert result["output_config"]["effort"] == effort @@ -1609,7 +1604,7 @@ def test_effort_validation(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) @@ -1618,18 +1613,14 @@ def test_effort_with_claude_opus_45(): config = AnthropicConfig() messages = [{"role": "user", "content": "Complex analysis task"}] - optional_params = { - "output_config": { - "effort": "high" - } - } + optional_params = {"output_config": {"effort": "high"}} result = config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert "output_config" in result @@ -1650,7 +1641,7 @@ def test_effort_validation_with_opus_46(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) assert result["output_config"]["effort"] == effort @@ -1661,14 +1652,16 @@ def test_max_effort_rejected_for_opus_45(): messages = [{"role": "user", "content": "Test"}] - with pytest.raises(ValueError, match="effort='max' is only supported by Claude Opus 4.6"): + with pytest.raises( + ValueError, match="effort='max' is only supported by Claude Opus 4.6" + ): optional_params = {"output_config": {"effort": "max"}} config.transform_request( model="claude-opus-4-5-20251101", messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) @@ -1685,23 +1678,16 @@ def test_effort_with_other_features(): "description": "Get data", "parameters": { "type": "object", - "properties": { - "query": {"type": "string"} - }, - "required": ["query"] - } - } + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, } ] optional_params = { - "output_config": { - "effort": "low" - }, + "output_config": {"effort": "low"}, "tools": tools, - "thinking": { - "type": "enabled", - "budget_tokens": 1000 - } + "thinking": {"type": "enabled", "budget_tokens": 1000}, } result = config.transform_request( @@ -1709,7 +1695,7 @@ def test_effort_with_other_features(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) # Verify all features are present @@ -1752,11 +1738,14 @@ def test_translate_system_message_skips_empty_list_content(): # Test list content with empty text block messages = [ - {"role": "system", "content": [ - {"type": "text", "text": ""}, - {"type": "text", "text": "Valid content"}, - {"type": "text", "text": ""}, - ]}, + { + "role": "system", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": "Valid content"}, + {"type": "text", "text": ""}, + ], + }, {"role": "user", "content": "Hello"}, ] @@ -1794,9 +1783,16 @@ def test_translate_system_message_preserves_cache_control(): # Test list content with cache_control messages = [ - {"role": "system", "content": [ - {"type": "text", "text": "Cached content", "cache_control": {"type": "ephemeral"}}, - ]}, + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Cached content", + "cache_control": {"type": "ephemeral"}, + }, + ], + }, {"role": "user", "content": "Hello"}, ] @@ -1938,7 +1934,7 @@ def test_transform_request_uses_dynamic_max_tokens(): messages=messages, optional_params={}, # No max_tokens provided litellm_params={}, - headers={} + headers={}, ) assert result["max_tokens"] == 64000 @@ -1959,7 +1955,7 @@ def test_transform_request_respects_user_max_tokens(): messages=messages, optional_params={"max_tokens": 1000}, litellm_params={}, - headers={} + headers={}, ) assert result["max_tokens"] == 1000 @@ -2006,11 +2002,12 @@ def test_calculate_usage_completion_tokens_details_with_reasoning(): "output_tokens": 500, } # Simulating reasoning content that would count as ~50 tokens - reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens + reasoning_content = ( + "Let me think about this step by step. " * 10 + ) # Roughly 50 tokens usage = config.calculate_usage( - usage_object=usage_object, - reasoning_content=reasoning_content + usage_object=usage_object, reasoning_content=reasoning_content ) # completion_tokens_details should be populated with both reasoning and text tokens @@ -2051,7 +2048,7 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): non_default_params=non_default_params, optional_params=optional_params, model=model, - drop_params=False + drop_params=False, ) # Should map to adaptive thinking type @@ -2062,7 +2059,9 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models(): # reasoning_effort should not be in the result (it's transformed to thinking) assert "reasoning_effort" not in result # Should set output_config with the mapped effort value - assert "output_config" in result, f"output_config missing for {model} with effort={effort}" + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort_map[effort] @@ -2123,10 +2122,10 @@ def test_reasoning_effort_maps_to_budget_thinking_for_non_opus_4_6(): # Test with Claude Sonnet 4.5 (non-Opus 4.6 model) test_cases = [ - ("low", 1024), # DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET - ("medium", 2048), # DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET - ("high", 4096), # DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET - ("minimal", 128), # DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET + ("low", 1024), # DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET + ("medium", 2048), # DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET + ("high", 4096), # DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET + ("minimal", 128), # DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET ] for effort, expected_budget in test_cases: @@ -2137,7 +2136,7 @@ def test_reasoning_effort_maps_to_budget_thinking_for_non_opus_4_6(): non_default_params=non_default_params, optional_params=optional_params, model="claude-sonnet-4-5-20250929", - drop_params=False + drop_params=False, ) # Should map to enabled thinking type with budget_tokens @@ -2166,9 +2165,9 @@ def test_reasoning_effort_sets_output_config_for_46_models(): drop_params=False, ) - assert "output_config" in result, ( - f"output_config missing for {model} with effort={effort}" - ) + assert ( + "output_config" in result + ), f"output_config missing for {model} with effort={effort}" assert result["output_config"]["effort"] == effort @@ -2207,9 +2206,9 @@ def test_reasoning_effort_does_not_set_output_config_for_older_models(): drop_params=False, ) - assert "output_config" not in result, ( - f"output_config should not be set for {model}" - ) + assert ( + "output_config" not in result + ), f"output_config should not be set for {model}" def test_max_effort_rejected_for_sonnet_46(): @@ -2217,7 +2216,9 @@ def test_max_effort_rejected_for_sonnet_46(): config = AnthropicConfig() messages = [{"role": "user", "content": "Test"}] - with pytest.raises(ValueError, match="effort='max' is only supported by Claude Opus 4.6"): + with pytest.raises( + ValueError, match="effort='max' is only supported by Claude Opus 4.6" + ): config.transform_request( model="claude-sonnet-4-6-20260219", messages=messages, @@ -2260,9 +2261,7 @@ def test_effort_beta_header_not_injected_for_46_models(): optional_params={"output_config": {"effort": "high"}}, model=model, ) - assert result is False, ( - f"is_effort_used should return False for {model}" - ) + assert result is False, f"is_effort_used should return False for {model}" def test_effort_beta_header_still_injected_for_older_models(): @@ -2302,17 +2301,12 @@ def test_code_execution_tool_results_extraction(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "I'll calculate that for you." - }, + {"type": "text", "text": "I'll calculate that for you."}, { "type": "server_tool_use", "id": "srvtoolu_01ABC", "name": "bash_code_execution", - "input": { - "command": "python3 << 'EOF'\nprint(2 + 2)\nEOF\n" - } + "input": {"command": "python3 << 'EOF'\nprint(2 + 2)\nEOF\n"}, }, { "type": "bash_code_execution_tool_result", @@ -2321,8 +2315,8 @@ def test_code_execution_tool_results_extraction(): "type": "bash_code_execution_result", "stdout": "4\n", "stderr": "", - "return_code": 0 - } + "return_code": 0, + }, }, { "type": "server_tool_use", @@ -2331,28 +2325,22 @@ def test_code_execution_tool_results_extraction(): "input": { "command": "create", "path": "test.txt", - "file_text": "Hello" - } + "file_text": "Hello", + }, }, { "type": "text_editor_code_execution_tool_result", "tool_use_id": "srvtoolu_01DEF", "content": { "type": "text_editor_code_execution_result", - "is_file_update": False - } + "is_file_update": False, + }, }, - { - "type": "text", - "text": "Done!" - } + {"type": "text", "text": "Done!"}, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } # Create mock HTTP response @@ -2377,11 +2365,17 @@ def test_code_execution_tool_results_extraction(): # Verify first tool call assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC" - assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[0].function.name + == "bash_code_execution" + ) # Verify second tool call assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF" - assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution" + assert ( + transformed_response.choices[0].message.tool_calls[1].function.name + == "text_editor_code_execution" + ) # Verify tool results are in provider_specific_fields provider_fields = transformed_response.choices[0].message.provider_specific_fields @@ -2404,10 +2398,83 @@ def test_code_execution_tool_results_extraction(): assert editor_result["content"]["is_file_update"] is False # Verify text content is properly concatenated - assert "I'll calculate that for you." in transformed_response.choices[0].message.content + assert ( + "I'll calculate that for you." + in transformed_response.choices[0].message.content + ) assert "Done!" in transformed_response.choices[0].message.content +def test_code_execution_tool_results_in_hidden_params(): + """ + Test that tool_results reaches _hidden_params so the Responses API adapter + can surface them via provider_specific_fields. + + The Responses API adapter reads _hidden_params.get("provider_specific_fields") + to set provider_specific_fields on the response. Without this, server-side + code execution results (stdout/stderr) are lost when using responses.create(). + """ + import httpx + + from litellm.types.utils import ModelResponse + + config = AnthropicConfig() + + mock_anthropic_response = { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [ + {"type": "text", "text": "Here's the result."}, + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC", + "name": "bash_code_execution", + "input": {"command": "echo hello"}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01ABC", + "content": { + "type": "bash_code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + ], + "stop_reason": "stop", + "stop_sequence": None, + "usage": {"input_tokens": 100, "output_tokens": 50}, + } + + mock_raw_response = MagicMock(spec=httpx.Response) + mock_raw_response.json.return_value = mock_anthropic_response + mock_raw_response.status_code = 200 + mock_raw_response.headers = {} + + model_response = ModelResponse() + + transformed_response = config.transform_parsed_response( + completion_response=mock_anthropic_response, + raw_response=mock_raw_response, + model_response=model_response, + json_mode=False, + prefix_prompt=None, + ) + + # Verify tool_results is in _hidden_params for the Responses API adapter + hidden = transformed_response._hidden_params + assert "provider_specific_fields" in hidden + assert "tool_results" in hidden["provider_specific_fields"] + assert len(hidden["provider_specific_fields"]["tool_results"]) == 1 + assert ( + hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] + == "hello\n" + ) + + def test_tool_search_tool_result_not_in_tool_results(): """ Test that tool_search_tool_result is NOT included in tool_results @@ -2425,21 +2492,12 @@ def test_tool_search_tool_result_not_in_tool_results(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "Found tools." - }, - { - "type": "tool_search_tool_result", - "tool_references": ["tool1", "tool2"] - } + {"type": "text", "text": "Found tools."}, + {"type": "tool_search_tool_result", "tool_references": ["tool1", "tool2"]}, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } mock_raw_response = MagicMock(spec=httpx.Response) @@ -2479,22 +2537,16 @@ def test_web_search_tool_result_backwards_compatibility(): "role": "assistant", "model": "claude-sonnet-4-5-20250929", "content": [ - { - "type": "text", - "text": "Here are the results." - }, + {"type": "text", "text": "Here are the results."}, { "type": "web_search_tool_result", "search_query": "test query", - "results": [{"title": "Result 1", "url": "https://example.com"}] - } + "results": [{"title": "Result 1", "url": "https://example.com"}], + }, ], "stop_reason": "stop", "stop_sequence": None, - "usage": { - "input_tokens": 100, - "output_tokens": 50 - } + "usage": {"input_tokens": 100, "output_tokens": 50}, } mock_raw_response = MagicMock(spec=httpx.Response) @@ -2540,24 +2592,28 @@ def test_compaction_block_extraction(): "content": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", }, { "type": "text", - "text": "I don't have access to real-time data, so I can't provide the current weather in San Francisco." - } + "text": "I don't have access to real-time data, so I can't provide the current weather in San Francisco.", + }, ], "stop_reason": "max_tokens", "stop_sequence": None, - "usage": { - "input_tokens": 86, - "output_tokens": 100 - } + "usage": {"input_tokens": 86, "output_tokens": 100}, } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify compaction blocks are extracted assert compaction_blocks is not None @@ -2587,18 +2643,12 @@ def test_compaction_block_in_provider_specific_fields(): "content": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", }, - { - "type": "text", - "text": "Here is the response." - } + {"type": "text", "text": "Here is the response."}, ], "stop_reason": "end_turn", - "usage": { - "input_tokens": 50, - "output_tokens": 25 - } + "usage": {"input_tokens": 50, "output_tokens": 25}, } raw_response = httpx.Response(status_code=200, headers={}) @@ -2618,7 +2668,10 @@ def test_compaction_block_in_provider_specific_fields(): assert "compaction_blocks" in provider_fields assert len(provider_fields["compaction_blocks"]) == 1 assert provider_fields["compaction_blocks"][0]["type"] == "compaction" - assert "Summary of the conversation" in provider_fields["compaction_blocks"][0]["content"] + assert ( + "Summary of the conversation" + in provider_fields["compaction_blocks"][0]["content"] + ) def test_multiple_compaction_blocks(): @@ -2629,24 +2682,22 @@ def test_multiple_compaction_blocks(): completion_response = { "content": [ - { - "type": "compaction", - "content": "First summary..." - }, - { - "type": "text", - "text": "Some text." - }, - { - "type": "compaction", - "content": "Second summary..." - } + {"type": "compaction", "content": "First summary..."}, + {"type": "text", "text": "Some text."}, + {"type": "compaction", "content": "Second summary..."}, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify both compaction blocks are extracted assert compaction_blocks is not None @@ -2665,37 +2716,26 @@ def test_compaction_block_request_transformation(): ) messages = [ - { - "role": "user", - "content": "What is the weather in San Francisco?" - }, + {"role": "user", "content": "What is the weather in San Francisco?"}, { "role": "assistant", "content": [ - { - "type": "text", - "text": "I don't have access to real-time data." - } + {"type": "text", "text": "I don't have access to real-time data."} ], "provider_specific_fields": { "compaction_blocks": [ { "type": "compaction", - "content": "Summary of the conversation: The user requested help building a web scraper..." + "content": "Summary of the conversation: The user requested help building a web scraper...", } ] - } + }, }, - { - "role": "user", - "content": "What about New York?" - } + {"role": "user", "content": "What about New York?"}, ] result = anthropic_messages_pt( - messages=messages, - model="claude-opus-4-6", - llm_provider="anthropic" + messages=messages, model="claude-opus-4-6", llm_provider="anthropic" ) # Find the assistant message @@ -2727,14 +2767,8 @@ def test_compaction_with_context_management(): messages = [{"role": "user", "content": "Hello"}] optional_params = { - "context_management": { - "edits": [ - { - "type": "compact_20260112" - } - ] - }, - "max_tokens": 100 + "context_management": {"edits": [{"type": "compact_20260112"}]}, + "max_tokens": 100, } result = config.transform_request( @@ -2742,7 +2776,7 @@ def test_compaction_with_context_management(): messages=messages, optional_params=optional_params, litellm_params={}, - headers={} + headers={}, ) # Verify context_management is included @@ -2758,30 +2792,28 @@ def test_compaction_block_with_other_content_types(): completion_response = { "content": [ - { - "type": "compaction", - "content": "Summary of previous conversation..." - }, - { - "type": "thinking", - "thinking": "Let me think about this..." - }, - { - "type": "text", - "text": "Based on my analysis..." - }, + {"type": "compaction", "content": "Summary of previous conversation..."}, + {"type": "thinking", "thinking": "Let me think about this..."}, + {"type": "text", "text": "Based on my analysis..."}, { "type": "tool_use", "id": "toolu_123", "name": "get_weather", - "input": {"location": "San Francisco"} - } + "input": {"location": "San Francisco"}, + }, ] } - text, citations, thinking_blocks, reasoning_content, tool_calls, web_search_results, tool_results, compaction_blocks = config.extract_response_content( - completion_response - ) + ( + text, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = config.extract_response_content(completion_response) # Verify all content types are extracted assert compaction_blocks is not None @@ -2798,11 +2830,11 @@ def test_map_openai_context_management_to_anthropic(): Test mapping OpenAI Responses API context_management format to Anthropic format. """ config = AnthropicConfig() - + # Test OpenAI list format with compaction openai_format = [{"type": "compaction", "compact_threshold": 200000}] result = config.map_openai_context_management_to_anthropic(openai_format) - + assert result is not None assert "edits" in result assert len(result["edits"]) == 1 @@ -2811,26 +2843,32 @@ def test_map_openai_context_management_to_anthropic(): assert result["edits"][0]["trigger"]["value"] == 200000 # Test OpenAI format with instructions - openai_format_with_instructions = [{ - "type": "compaction", - "compact_threshold": 150000, - "instructions": "Focus on preserving code snippets" - }] - result = config.map_openai_context_management_to_anthropic(openai_format_with_instructions) - + openai_format_with_instructions = [ + { + "type": "compaction", + "compact_threshold": 150000, + "instructions": "Focus on preserving code snippets", + } + ] + result = config.map_openai_context_management_to_anthropic( + openai_format_with_instructions + ) + assert result is not None assert result["edits"][0]["trigger"]["value"] == 150000 assert result["edits"][0]["instructions"] == "Focus on preserving code snippets" - + # Test Anthropic format (should pass through) anthropic_format = { - "edits": [{ - "type": "compact_20260112", - "trigger": {"type": "input_tokens", "value": 150000} - }] + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + } + ] } result = config.map_openai_context_management_to_anthropic(anthropic_format) - + assert result == anthropic_format @@ -2839,46 +2877,51 @@ def test_map_openai_params_with_context_management(): Test that map_openai_params correctly transforms context_management from OpenAI to Anthropic format. """ config = AnthropicConfig() - + # Test with OpenAI list format non_default_params = { "context_management": [{"type": "compaction", "compact_threshold": 200000}] } optional_params = {} - + result = config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) - + assert "context_management" in result assert "edits" in result["context_management"] assert result["context_management"]["edits"][0]["type"] == "compact_20260112" assert result["context_management"]["edits"][0]["trigger"]["value"] == 200000 - + # Test with Anthropic dict format (should pass through) non_default_params_anthropic = { "context_management": { - "edits": [{ - "type": "compact_20260112", - "trigger": {"type": "input_tokens", "value": 150000}, - "instructions": "Focus on preserving code" - }] + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + "instructions": "Focus on preserving code", + } + ] } } optional_params = {} - + result = config.map_openai_params( non_default_params=non_default_params_anthropic, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) - + assert "context_management" in result - assert result["context_management"] == non_default_params_anthropic["context_management"] + assert ( + result["context_management"] + == non_default_params_anthropic["context_management"] + ) def test_cache_control_in_supported_params(): @@ -2897,9 +2940,7 @@ def test_map_openai_params_with_cache_control(): """ config = AnthropicConfig() - non_default_params = { - "cache_control": {"type": "ephemeral"} - } + non_default_params = {"cache_control": {"type": "ephemeral"}} optional_params = {} result = config.map_openai_params( @@ -2919,9 +2960,7 @@ def test_map_openai_params_cache_control_ignored_when_not_dict(): """ config = AnthropicConfig() - non_default_params = { - "cache_control": "ephemeral" - } + non_default_params = {"cache_control": "ephemeral"} optional_params = {} result = config.map_openai_params( @@ -2974,17 +3013,9 @@ def test_compaction_block_empty_list_not_added(): "type": "message", "role": "assistant", "model": "claude-opus-4-6", - "content": [ - { - "type": "text", - "text": "Just a regular response." - } - ], + "content": [{"type": "text", "text": "Just a regular response."}], "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 5 - } + "usage": {"input_tokens": 10, "output_tokens": 5}, } raw_response = httpx.Response(status_code=200, headers={}) @@ -3001,7 +3032,10 @@ def test_compaction_block_empty_list_not_added(): # Verify compaction_blocks is not in provider_specific_fields when there are none provider_fields = result.choices[0].message.provider_specific_fields if provider_fields: - assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None + assert ( + "compaction_blocks" not in provider_fields + or provider_fields.get("compaction_blocks") is None + ) def test_fast_mode_beta_header(): @@ -3014,8 +3048,7 @@ def test_fast_mode_beta_header(): optional_params = {"speed": "fast"} result_headers = config.update_headers_with_optional_anthropic_beta( - headers=headers, - optional_params=optional_params + headers=headers, optional_params=optional_params ) assert "anthropic-beta" in result_headers @@ -3029,14 +3062,10 @@ def test_fast_mode_with_other_beta_headers(): config = AnthropicConfig() headers = {} - optional_params = { - "speed": "fast", - "output_format": {"type": "json_object"} - } + optional_params = {"speed": "fast", "output_format": {"type": "json_object"}} result_headers = config.update_headers_with_optional_anthropic_beta( - headers=headers, - optional_params=optional_params + headers=headers, optional_params=optional_params ) assert "anthropic-beta" in result_headers @@ -3056,9 +3085,7 @@ def test_fast_mode_usage_calculation(): } usage = config.calculate_usage( - usage_object=usage_object, - reasoning_content=None, - speed="fast" + usage_object=usage_object, reasoning_content=None, speed="fast" ) assert usage.prompt_tokens == 1000 @@ -3171,7 +3198,7 @@ def test_fast_mode_parameter_mapping(): non_default_params=non_default_params, optional_params=optional_params, model="claude-opus-4-6", - drop_params=False + drop_params=False, ) assert "speed" in result @@ -3236,9 +3263,9 @@ def test_map_tool_helper_enforces_object_type_when_missing(): assert "properties" in result["input_schema"] assert "query" in result["input_schema"]["properties"] # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_enforces_object_type_when_wrong_type(): @@ -3264,13 +3291,13 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type(): result, _ = config._map_tool_helper(tool) assert result is not None assert result["input_schema"]["type"] == "object" - assert result["input_schema"].get("properties") == {}, ( - "properties should be injected as {} when schema has non-object type and no properties key" - ) + assert ( + result["input_schema"].get("properties") == {} + ), "properties should be injected as {} when schema has non-object type and no properties key" # Original parameters dict must not be modified in place - assert tool["function"]["parameters"] == original_params, ( - "parameters dict was mutated; _map_tool_helper should not modify caller data" - ) + assert ( + tool["function"]["parameters"] == original_params + ), "parameters dict was mutated; _map_tool_helper should not modify caller data" def test_map_tool_helper_preserves_valid_object_schema(): From 2bf8751f6b0cfe30b51e4bb298c7e36ebc4ea46b Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 17:57:25 +0100 Subject: [PATCH 07/57] fix: streaming code_interpreter_results dropped for multiple code executions stream_chunk_builder uses "last value wins" for list-valued provider_specific_fields keys. _build_code_interpreter_results was emitting only new items (incremental), so earlier results were silently dropped when multiple sequential code executions occurred. - Emit cumulative list from _build_code_interpreter_results, matching web_search_results pattern - Assemble server_tool_use input from input_json_delta deltas at content_block_stop (Anthropic streams input: {} in start block) - Handle dict items in _extract_tool_result_output_items after model_dump() serialization in stream_chunk_builder - Simplify _merge_provider_specific_fields to last-value-wins for lists, matching stream_chunk_builder semantics --- litellm/llms/anthropic/chat/handler.py | 45 ++++-- .../streaming_iterator.py | 19 ++- .../transformation.py | 5 +- .../chat/test_anthropic_chat_handler.py | 133 +++++++++++++++--- 4 files changed, 165 insertions(+), 37 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 51b9c9835a..7d5fa2a559 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -545,7 +545,7 @@ class ModelResponseIterator: # Track server tool use inputs and results for code_interpreter_results self._server_tool_inputs: Dict[str, Any] = {} self.tool_results: List[Dict[str, Any]] = [] - self._last_code_interpreter_results_count: int = 0 + self._current_server_tool_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -695,13 +695,14 @@ class ModelResponseIterator: Called during streaming to produce provider-neutral code_interpreter_results alongside the raw tool_results, so the Responses API layer doesn't need Anthropic-specific knowledge. + + Returns the full cumulative list each time (not incremental), matching + how web_search_results works. stream_chunk_builder uses "last value + wins" for list-valued provider_specific_fields keys, so the last + emission must contain every result. """ - # Only convert tool_results added since the last call to avoid - # duplicates when _merge_provider_specific_fields extends the list. - new_results = self.tool_results[self._last_code_interpreter_results_count :] - self._last_code_interpreter_results_count = len(self.tool_results) results = [] - for tr in new_results: + for tr in self.tool_results: call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): @@ -793,17 +794,23 @@ class ModelResponseIterator: ), index=self.tool_index, ) - # Track server tool use inputs for code_interpreter_results + # Track server tool use inputs for code_interpreter_results. + # The initial input in content_block_start is typically {} + # for streaming; the full input arrives via input_json_delta + # and is assembled at content_block_stop. if ( content_block_start["content_block"]["type"] == "server_tool_use" ): + self._current_server_tool_id = content_block_start[ + "content_block" + ]["id"] tool_input = content_block_start["content_block"].get( "input", {} ) - self._server_tool_inputs[ - content_block_start["content_block"]["id"] - ] = tool_input + self._server_tool_inputs[self._current_server_tool_id] = ( + tool_input + ) # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -886,6 +893,24 @@ class ModelResponseIterator: ), index=self.tool_index, ) + # Update server_tool_inputs with fully assembled input + # from input_json_delta chunks (content_block_start has {}) + if ( + self.current_content_block_type == "server_tool_use" + and self._current_server_tool_id + ): + args = "" + for block in self.content_blocks: + if block["delta"]["type"] == "input_json_delta": + args += block["delta"].get("partial_json", "") + if args: + try: + self._server_tool_inputs[ + self._current_server_tool_id + ] = json.loads(args) + except (json.JSONDecodeError, TypeError): + pass + self._current_server_tool_id = None # Reset response_format tool tracking when block stops self.is_response_format_tool = False # Reset current content block type diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 0b7d6e8a7a..0672b03bcd 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -481,17 +481,16 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return event def _merge_provider_specific_fields(self, src: dict) -> None: - """Merge provider_specific_fields, extending list values instead of replacing.""" + """Merge provider_specific_fields using last-value-wins for lists. + + List-valued keys (web_search_results, tool_results, + code_interpreter_results, etc.) are emitted cumulatively — each + emission contains the full list so far. Using "last value wins" + matches stream_chunk_builder's semantics and avoids quadratic + growth from repeated extend calls. + """ for key, val in src.items(): - existing = self._accumulated_provider_specific_fields.get(key) - if ( - existing is not None - and isinstance(val, list) - and isinstance(existing, list) - ): - existing.extend(val) - else: - self._accumulated_provider_specific_fields[key] = val + self._accumulated_provider_specific_fields[key] = val def create_litellm_model_response(self) -> Optional[ModelResponse]: response = cast( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b54d5930ef..b7f7e9adda 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1738,7 +1738,10 @@ class LiteLLMCompletionResponsesConfig: ) ) if tool_result_items: - result_by_id = {item.id: item for item in tool_result_items} + result_by_id = { + (item.get("id") if isinstance(item, dict) else item.id): item + for item in tool_result_items + } replaced_ids = set(result_by_id.keys()) responses_output = [ ( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index 35c7a62027..d7a04a054a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1245,9 +1245,9 @@ def test_streaming_code_execution_produces_code_interpreter_results(): def test_streaming_multiple_code_executions_no_duplicates(): """ - Test that multiple code executions in a single streaming response produce - exactly one code_interpreter_result per execution — no duplicates from - _build_code_interpreter_results rebuilding the full list. + Test that multiple code executions in a single streaming response emit + cumulative code_interpreter_results on each chunk (matching stream_chunk_builder's + "last value wins" contract). The final emission must contain ALL results. """ chunks = [ { @@ -1323,24 +1323,125 @@ def test_streaming_multiple_code_executions_no_duplicates(): iterator = ModelResponseIterator(None, sync_stream=True) - # Collect ALL code_interpreter_results emitted across all chunks - all_results = [] + # Collect each emission of code_interpreter_results + emissions = [] for chunk in chunks: parsed = iterator.chunk_parser(chunk) psf = None if parsed.choices and parsed.choices[0].delta: psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) if psf and "code_interpreter_results" in psf: - all_results.extend(psf["code_interpreter_results"]) + emissions.append(psf["code_interpreter_results"]) - # Should have exactly 2 results, one per execution — no duplicates - assert len(all_results) == 2, ( - f"Expected 2 code_interpreter_results, got {len(all_results)}. " - f"IDs: {[r.id for r in all_results]}" + # Should have 2 emissions (one per tool_result block) + assert len(emissions) == 2, f"Expected 2 emissions, got {len(emissions)}" + + # First emission: cumulative list with 1 result + assert len(emissions[0]) == 1 + assert emissions[0][0].id == "srvtoolu_01AAA" + assert emissions[0][0].code == "echo first" + assert emissions[0][0].outputs[0].logs == "first\n" + + # Second (final) emission: cumulative list with BOTH results + # This is what stream_chunk_builder will pick as "last value wins" + assert len(emissions[1]) == 2, ( + f"Expected final emission to have 2 results, got {len(emissions[1])}. " + f"IDs: {[r.id for r in emissions[1]]}" ) - assert all_results[0].id == "srvtoolu_01AAA" - assert all_results[0].code == "echo first" - assert all_results[0].outputs[0].logs == "first\n" - assert all_results[1].id == "srvtoolu_01BBB" - assert all_results[1].code == "echo second" - assert all_results[1].outputs[0].logs == "second\n" + assert emissions[1][0].id == "srvtoolu_01AAA" + assert emissions[1][0].code == "echo first" + assert emissions[1][0].outputs[0].logs == "first\n" + assert emissions[1][1].id == "srvtoolu_01BBB" + assert emissions[1][1].code == "echo second" + assert emissions[1][1].outputs[0].logs == "second\n" + + +def test_streaming_code_execution_input_assembled_from_deltas(): + """ + In real Anthropic streaming, content_block_start for server_tool_use has + input: {}. The actual input arrives via input_json_delta deltas and must + be assembled at content_block_stop so the code field is populated. + + This test uses realistic chunk shapes (empty input in start, partial JSON + in deltas) to exercise the input assembly path. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + # server_tool_use with empty input (real streaming behaviour) + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "code_execution", + "input": {}, + }, + }, + # Input arrives via deltas, split across two chunks + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": '{"comma', + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": 'nd": "echo hello"}', + }, + }, + {"type": "content_block_stop", "index": 0}, + # Tool result + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "code_execution_result", + "stdout": "hello\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + # The code field must contain the assembled input, not be empty + assert code_results is not None, "No code_interpreter_results emitted" + assert len(code_results) == 1 + assert code_results[0].id == "srvtoolu_01AAA" + assert code_results[0].code == "echo hello" + assert code_results[0].outputs[0].logs == "hello\n" From 4be1d76fd7da5c4c0b04f1ead59e98bcf54d1dfb Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 18:37:32 +0100 Subject: [PATCH 08/57] fix: empty stdout/stderr produces str(content) instead of empty logs When both stdout and stderr are empty strings, the `if parts else str(content)` fallback produced the raw dict representation as logs. Drop the fallback so logs is correctly empty. --- litellm/llms/anthropic/chat/handler.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 7d5fa2a559..88d9ee6596 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -711,7 +711,7 @@ class ModelResponseIterator: parts.append(content["stdout"]) if content.get("stderr"): parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) if parts else str(content) + logs = "".join(parts) else: logs = str(content) tool_input = self._server_tool_inputs.get(call_id, {}) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 033afea2ff..21ca8db825 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1775,7 +1775,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): parts.append(content["stdout"]) if content.get("stderr"): parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) if parts else str(content) + logs = "".join(parts) else: logs = str(content) code_interpreter_results.append( From 5b3e84f383627c951c4d7bf5f150e4173c39fdc4 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 18:59:47 +0100 Subject: [PATCH 09/57] fix: address remaining review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Empty stdout/stderr now produces outputs=None (matching OpenAI parity) instead of outputs=[{logs:""}], in both streaming and non-streaming paths - Fix test fixture to use real Anthropic type "bash_code_execution_tool_result" instead of "code_execution_tool_result" - Add test for empty-output → outputs=None behavior - Add unit tests for _extract_tool_result_output_items: Pydantic objects, plain dicts (post-model_dump), empty/missing provider_specific_fields, and in-place substitution preserving output ordering --- litellm/llms/anthropic/chat/handler.py | 5 +- litellm/llms/anthropic/chat/transformation.py | 9 +- .../chat/test_anthropic_chat_handler.py | 72 +++++++- ...est_code_interpreter_results_extraction.py | 163 ++++++++++++++++++ 4 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 88d9ee6596..0b84830a8b 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -716,6 +716,9 @@ class ModelResponseIterator: logs = str(content) tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" + log_outputs = ( + [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None + ) results.append( OutputCodeInterpreterCall( type="code_interpreter_call", @@ -723,7 +726,7 @@ class ModelResponseIterator: code=code, container_id=None, status="completed", - outputs=[OutputCodeInterpreterCallLog(type="logs", logs=logs)], + outputs=log_outputs, ) ) return results diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 21ca8db825..99b02e7ab4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1778,6 +1778,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): logs = "".join(parts) else: logs = str(content) + log_outputs = ( + [OutputCodeInterpreterCallLog(type="logs", logs=logs)] + if logs + else None + ) code_interpreter_results.append( OutputCodeInterpreterCall( type="code_interpreter_call", @@ -1785,9 +1790,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code=code_by_id.get(call_id, ""), container_id=container_id, status="completed", - outputs=[ - OutputCodeInterpreterCallLog(type="logs", logs=logs) - ], + outputs=log_outputs, ) ) provider_specific_fields["code_interpreter_results"] = ( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index d7a04a054a..ab298f6809 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1410,10 +1410,10 @@ def test_streaming_code_execution_input_assembled_from_deltas(): "type": "content_block_start", "index": 1, "content_block": { - "type": "code_execution_tool_result", + "type": "bash_code_execution_tool_result", "tool_use_id": "srvtoolu_01AAA", "content": { - "type": "code_execution_result", + "type": "bash_code_execution_result", "stdout": "hello\n", "stderr": "", "return_code": 0, @@ -1445,3 +1445,71 @@ def test_streaming_code_execution_input_assembled_from_deltas(): assert code_results[0].id == "srvtoolu_01AAA" assert code_results[0].code == "echo hello" assert code_results[0].outputs[0].logs == "hello\n" + + +def test_empty_output_produces_null_outputs(): + """ + When both stdout and stderr are empty, outputs should be None + (matching OpenAI's native behavior) rather than [{logs: ""}]. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {"command": "true"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + assert code_results is not None, "No code_interpreter_results emitted" + assert len(code_results) == 1 + assert code_results[0].id == "srvtoolu_01AAA" + assert ( + code_results[0].outputs is None + ), f"Expected outputs=None for empty execution, got {code_results[0].outputs}" diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py new file mode 100644 index 0000000000..c6ff1c7af8 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -0,0 +1,163 @@ +""" +Tests for the Responses API _extract_tool_result_output_items path +and the non-streaming _hidden_params propagation of code_interpreter_results. +""" + +from unittest.mock import MagicMock + +from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, +) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + OutputCodeInterpreterCallLog, +) +from litellm.types.utils import Choices, Message, ModelResponse + + +def _make_model_response(code_interpreter_results=None, provider_specific_fields=None): + """Helper to build a ModelResponse with provider_specific_fields on the message.""" + psf = provider_specific_fields or {} + if code_interpreter_results is not None: + psf["code_interpreter_results"] = code_interpreter_results + msg = Message(content="test", provider_specific_fields=psf if psf else None) + choice = Choices(index=0, message=msg, finish_reason="stop") + resp = ModelResponse() + resp.choices = [choice] + return resp + + +def test_extract_tool_result_output_items_from_pydantic_objects(): + """Non-streaming path: code_interpreter_results are Pydantic OutputCodeInterpreterCall objects.""" + items = [ + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01AAA", + code="echo hello", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="hello\n")], + ), + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01BBB", + code="echo world", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="world\n")], + ), + ] + resp = _make_model_response(code_interpreter_results=items) + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert len(result) == 2 + assert result[0].id == "srvtoolu_01AAA" + assert result[1].id == "srvtoolu_01BBB" + + +def test_extract_tool_result_output_items_from_dicts(): + """Streaming path: after model_dump(), code_interpreter_results are plain dicts.""" + items = [ + { + "type": "code_interpreter_call", + "id": "srvtoolu_01AAA", + "code": "echo hello", + "container_id": None, + "status": "completed", + "outputs": [{"type": "logs", "logs": "hello\n"}], + }, + ] + resp = _make_model_response(code_interpreter_results=items) + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert len(result) == 1 + assert result[0]["id"] == "srvtoolu_01AAA" + + +def test_extract_tool_result_output_items_empty(): + """No code_interpreter_results → empty list.""" + resp = _make_model_response() + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert result == [] + + +def test_extract_tool_result_output_items_no_provider_specific_fields(): + """Message with no provider_specific_fields → empty list.""" + msg = Message(content="test") + choice = Choices(index=0, message=msg, finish_reason="stop") + resp = ModelResponse() + resp.choices = [choice] + result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + assert result == [] + + +def test_in_place_substitution_preserves_ordering(): + """ + function_call items matching code_interpreter_results should be replaced + in-place, preserving the original output ordering. + + Simulates: [message, function_call(exec1), function_call(regular), function_call(exec2)] + Expected: [message, code_interpreter_call(exec1), function_call(regular), code_interpreter_call(exec2)] + """ + code_results = [ + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01AAA", + code="echo first", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="first\n")], + ), + OutputCodeInterpreterCall( + type="code_interpreter_call", + id="srvtoolu_01CCC", + code="echo third", + container_id=None, + status="completed", + outputs=[OutputCodeInterpreterCallLog(type="logs", logs="third\n")], + ), + ] + resp = _make_model_response(code_interpreter_results=code_results) + + # Build a mock responses_output list with interleaved items + class MockItem: + def __init__(self, type, call_id=None): + self.type = type + self.call_id = call_id + + msg_item = MockItem(type="message") + fc_exec1 = MockItem(type="function_call", call_id="srvtoolu_01AAA") + fc_regular = MockItem(type="function_call", call_id="srvtoolu_01BBB") + fc_exec2 = MockItem(type="function_call", call_id="srvtoolu_01CCC") + + responses_output = [msg_item, fc_exec1, fc_regular, fc_exec2] + + # Apply the same logic as _transform_chat_completion_choices_to_responses_output + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) + ) + if tool_result_items: + result_by_id = { + (item.get("id") if isinstance(item, dict) else item.id): item + for item in tool_result_items + } + replaced_ids = set(result_by_id.keys()) + responses_output = [ + ( + result_by_id[getattr(item, "call_id", None)] + if ( + getattr(item, "type", None) == "function_call" + and getattr(item, "call_id", None) in replaced_ids + ) + else item + ) + for item in responses_output + ] + + # Verify ordering: message, code_interpreter(AAA), function_call(BBB), code_interpreter(CCC) + assert len(responses_output) == 4 + assert responses_output[0].type == "message" + assert responses_output[1].type == "code_interpreter_call" + assert responses_output[1].id == "srvtoolu_01AAA" + assert responses_output[2].type == "function_call" + assert responses_output[2].call_id == "srvtoolu_01BBB" + assert responses_output[3].type == "code_interpreter_call" + assert responses_output[3].id == "srvtoolu_01CCC" From 3962fbc33ac12c8a6e232dea488779f5570cb5f2 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Tue, 17 Mar 2026 19:26:03 +0100 Subject: [PATCH 10/57] fix: non-dict tool result content falls back to outputs=None Replace str(content) fallback with empty string so non-dict content (e.g. list-shaped text_editor results) produces outputs=None instead of raw Python object representations in logs. --- litellm/llms/anthropic/chat/handler.py | 2 +- litellm/llms/anthropic/chat/transformation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 0b84830a8b..387901588f 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -713,7 +713,7 @@ class ModelResponseIterator: parts.append(f"STDERR: {content['stderr']}") logs = "".join(parts) else: - logs = str(content) + logs = "" tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" log_outputs = ( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 99b02e7ab4..238d72eda4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1777,7 +1777,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): parts.append(f"STDERR: {content['stderr']}") logs = "".join(parts) else: - logs = str(content) + logs = "" log_outputs = ( [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs From 8f60117228821ccde41d4845382afee507dfb70c Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 00:42:03 +0100 Subject: [PATCH 11/57] fix: guard code_interpreter conversion to bash_code_execution results only Skip non-bash tool result types (e.g. text_editor_code_execution_tool_result) to avoid producing empty code_interpreter_call items in Responses API output. --- litellm/llms/anthropic/chat/handler.py | 2 ++ litellm/llms/anthropic/chat/transformation.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 387901588f..91fd303406 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -703,6 +703,8 @@ class ModelResponseIterator: """ results = [] for tr in self.tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 238d72eda4..ca101df0e9 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1767,6 +1767,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): pass code_interpreter_results = [] for tr in tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) if isinstance(content, dict): From d10007cef49ae278e97c05e40ce367481e983975 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 11:15:15 +0100 Subject: [PATCH 12/57] test: add non-bash skip test and mock end-to-end streaming integration test - test_non_bash_tool_result_skipped: verifies text_editor results produce zero code_interpreter_call items - test_end_to_end_streaming_chunks_to_code_interpreter_output: exercises full path from Anthropic SSE chunks through ModelResponseIterator, stream_chunk_builder, and _extract_tool_result_output_items without a live server --- .../chat/test_anthropic_chat_handler.py | 68 +++++++++++ ...est_code_interpreter_results_extraction.py | 106 +++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index ab298f6809..20427e8cc9 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1513,3 +1513,71 @@ def test_empty_output_produces_null_outputs(): assert ( code_results[0].outputs is None ), f"Expected outputs=None for empty execution, got {code_results[0].outputs}" + + +def test_non_bash_tool_result_skipped(): + """ + Tool result types other than bash_code_execution_tool_result (e.g. + text_editor_code_execution_tool_result) should be skipped and NOT + produce code_interpreter_call items. + """ + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "text_editor", + "input": {"command": "view", "path": "/tmp/test.py"}, + }, + }, + {"type": "content_block_stop", "index": 0}, + # text_editor result — should NOT become a code_interpreter_call + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "text_editor_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": [ + {"type": "text", "text": "file contents here"}, + ], + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + + code_results = None + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + psf = None + if parsed.choices and parsed.choices[0].delta: + psf = getattr(parsed.choices[0].delta, "provider_specific_fields", None) + if psf and "code_interpreter_results" in psf: + code_results = psf["code_interpreter_results"] + + # code_interpreter_results should be emitted but empty (no bash results) + assert ( + code_results is not None + ), "Expected code_interpreter_results key to be emitted" + assert ( + len(code_results) == 0 + ), f"Expected 0 code_interpreter_results for text_editor result, got {len(code_results)}" diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py index c6ff1c7af8..eea9be38fa 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -1,10 +1,13 @@ """ -Tests for the Responses API _extract_tool_result_output_items path -and the non-streaming _hidden_params propagation of code_interpreter_results. +Tests for the Responses API _extract_tool_result_output_items path, +the non-streaming _hidden_params propagation of code_interpreter_results, +and mock end-to-end streaming integration. """ from unittest.mock import MagicMock +from litellm.llms.anthropic.chat.handler import ModelResponseIterator +from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -161,3 +164,102 @@ def test_in_place_substitution_preserves_ordering(): assert responses_output[2].call_id == "srvtoolu_01BBB" assert responses_output[3].type == "code_interpreter_call" assert responses_output[3].id == "srvtoolu_01CCC" + + +def test_end_to_end_streaming_chunks_to_code_interpreter_output(): + """ + Mock end-to-end test: Anthropic SSE chunks → ModelResponseIterator → + stream_chunk_builder → _extract_tool_result_output_items → final output + with code_interpreter_call items replacing function_call items. + + This exercises the full streaming data flow without a live server. + """ + # Realistic Anthropic streaming chunks for a single code execution + raw_chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_01XYZ", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 100, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "server_tool_use", + "id": "srvtoolu_01AAA", + "name": "bash_code_execution", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": '{"command": "echo e2e_test"}', + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01AAA", + "content": { + "type": "bash_code_execution_result", + "stdout": "e2e_test\n", + "stderr": "", + "return_code": 0, + }, + }, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + # Step 1: Parse chunks through ModelResponseIterator (Anthropic handler) + iterator = ModelResponseIterator(None, sync_stream=True) + parsed_chunks = [] + for chunk in raw_chunks: + parsed = iterator.chunk_parser(chunk) + d = parsed.model_dump() + # In production, CustomStreamWrapper sets the model on each chunk; + # stream_chunk_builder requires it. + d["model"] = "claude-sonnet-4-20250514" + parsed_chunks.append(d) + + # Step 2: Assemble via stream_chunk_builder (simulates end-of-stream) + assembled = stream_chunk_builder(chunks=parsed_chunks) + assert assembled is not None + + # Verify stream_chunk_builder picked up code_interpreter_results via last-value-wins + psf = assembled.choices[0].message.provider_specific_fields + assert psf is not None + assert "code_interpreter_results" in psf + code_results = psf["code_interpreter_results"] + assert len(code_results) == 1 + # After model_dump + stream_chunk_builder, results are plain dicts + assert code_results[0]["id"] == "srvtoolu_01AAA" + assert code_results[0]["code"] == "echo e2e_test" + + # Step 3: Extract via _extract_tool_result_output_items (Responses API layer) + tool_result_items = ( + LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(assembled) + ) + assert len(tool_result_items) == 1 + item = tool_result_items[0] + # Items are dicts after the model_dump path + assert item["type"] == "code_interpreter_call" + assert item["id"] == "srvtoolu_01AAA" + assert item["code"] == "echo e2e_test" + assert item["outputs"][0]["logs"] == "e2e_test\n" From cf8d1ac521648fea10bc121cd51da166e96493a4 Mon Sep 17 00:00:00 2001 From: Andrzej Pomirski Date: Wed, 18 Mar 2026 12:05:25 +0100 Subject: [PATCH 13/57] fix: streaming container_id and consistent Pydantic types in output - Populate container_id on streaming code_interpreter_results by re-emitting at message_delta when container info arrives - Reconstruct Pydantic OutputCodeInterpreterCall objects from plain dicts in _extract_tool_result_output_items so responses_output has uniform types across streaming and non-streaming paths --- litellm/llms/anthropic/chat/handler.py | 14 +++++++++++++- .../transformation.py | 14 +++++++++----- .../test_code_interpreter_results_extraction.py | 17 ++++++++++------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 91fd303406..70ecf91725 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -546,6 +546,7 @@ class ModelResponseIterator: self._server_tool_inputs: Dict[str, Any] = {} self.tool_results: List[Dict[str, Any]] = [] self._current_server_tool_id: Optional[str] = None + self._container_id: Optional[str] = None def check_empty_tool_call_args(self) -> bool: """ @@ -726,7 +727,7 @@ class ModelResponseIterator: type="code_interpreter_call", id=call_id, code=code, - container_id=None, + container_id=self._container_id, status="completed", outputs=log_outputs, ) @@ -928,6 +929,17 @@ class ModelResponseIterator: finish_reason, usage, container = self._handle_message_delta(chunk) if container: provider_specific_fields["container"] = container + # Store container_id and re-emit code_interpreter_results + # so stream_chunk_builder's last-value-wins picks up the + # version with container_id populated. + container_id = ( + container.get("id") if isinstance(container, dict) else None + ) + if container_id and self.tool_results: + self._container_id = container_id + provider_specific_fields["code_interpreter_results"] = ( + self._build_code_interpreter_results() + ) elif type_chunk == "message_start": """ Anthropic diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b7f7e9adda..cf18511bfa 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1738,10 +1738,7 @@ class LiteLLMCompletionResponsesConfig: ) ) if tool_result_items: - result_by_id = { - (item.get("id") if isinstance(item, dict) else item.id): item - for item in tool_result_items - } + result_by_id = {item.id: item for item in tool_result_items} replaced_ids = set(result_by_id.keys()) responses_output = [ ( @@ -1778,7 +1775,14 @@ class LiteLLMCompletionResponsesConfig: continue results = psf.get("code_interpreter_results") if results and isinstance(results, list): - output_items.extend(results) + for item in results: + # In the streaming path, items are plain dicts after + # model_dump() in stream_chunk_builder. Reconstruct + # Pydantic objects so responses_output has a uniform type. + if isinstance(item, dict): + output_items.append(OutputCodeInterpreterCall(**item)) + else: + output_items.append(item) return output_items @staticmethod diff --git a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py index eea9be38fa..60e45c9b8c 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py +++ b/tests/test_litellm/llms/anthropic/chat/test_code_interpreter_results_extraction.py @@ -58,7 +58,8 @@ def test_extract_tool_result_output_items_from_pydantic_objects(): def test_extract_tool_result_output_items_from_dicts(): - """Streaming path: after model_dump(), code_interpreter_results are plain dicts.""" + """Streaming path: after model_dump(), code_interpreter_results are plain dicts. + _extract_tool_result_output_items reconstructs them as Pydantic objects.""" items = [ { "type": "code_interpreter_call", @@ -72,7 +73,8 @@ def test_extract_tool_result_output_items_from_dicts(): resp = _make_model_response(code_interpreter_results=items) result = LiteLLMCompletionResponsesConfig._extract_tool_result_output_items(resp) assert len(result) == 1 - assert result[0]["id"] == "srvtoolu_01AAA" + assert isinstance(result[0], OutputCodeInterpreterCall) + assert result[0].id == "srvtoolu_01AAA" def test_extract_tool_result_output_items_empty(): @@ -258,8 +260,9 @@ def test_end_to_end_streaming_chunks_to_code_interpreter_output(): ) assert len(tool_result_items) == 1 item = tool_result_items[0] - # Items are dicts after the model_dump path - assert item["type"] == "code_interpreter_call" - assert item["id"] == "srvtoolu_01AAA" - assert item["code"] == "echo e2e_test" - assert item["outputs"][0]["logs"] == "e2e_test\n" + # Items are reconstructed as Pydantic OutputCodeInterpreterCall objects + assert isinstance(item, OutputCodeInterpreterCall) + assert item.type == "code_interpreter_call" + assert item.id == "srvtoolu_01AAA" + assert item.code == "echo e2e_test" + assert item.outputs[0].logs == "e2e_test\n" From b13a7c679033ceea1b93ca82440f5b6bf15a8bd7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:17:19 -0700 Subject: [PATCH 14/57] Fix guardrail_mode.replace crash when backend returns non-string value The backend type for guardrail_mode is Optional[Union[str, List[str], Dict]] but the UI typed it as just string, causing a crash when .replace() was called on null/object/array values. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../GuardrailViewer/GuardrailViewer.test.tsx | 29 ++++++++++++++++ .../GuardrailViewer/GuardrailViewer.tsx | 33 +++++++++++++++---- .../GuardrailViewer/__tests__/fixtures.ts | 2 +- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 28991ebfa0..60778a1337 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -151,6 +151,35 @@ describe("GuardrailViewer", () => { expect(screen.queryByText(/Raw Bedrock Guardrail Response/)).not.toBeInTheDocument(); }); + it("renders without crashing when guardrail_mode is null", () => { + const data = makeGuardrailInformation({ guardrail_mode: null }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + // Null mode should display as dash + expect(screen.getByText("—")).toBeInTheDocument(); + }); + + it("renders without crashing when guardrail_mode is an object", () => { + const data = makeGuardrailInformation({ + guardrail_mode: { default: "pre_call", tags: {} }, + }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + }); + + it("renders without crashing when guardrail_mode is an array", () => { + const data = makeGuardrailInformation({ + guardrail_mode: ["pre_call", "post_call"], + }); + renderWithProviders(); + + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + }); + it("integration: renders with real Bedrock details without mocks", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation({ diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1ab4744b89..0c37b70c8b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -40,7 +40,7 @@ interface GuardrailInformation { duration: number; end_time: number; start_time: number; - guardrail_mode: string; + guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse | any; @@ -77,9 +77,25 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ "litellm_content_filter", ]); -const formatMode = (mode: unknown): string => { - if (mode == null || mode === "") return "—"; - const s = typeof mode === "string" ? mode : String(mode); +/** + * Extracts a plain string from guardrail_mode, which may be a string, + * an array of strings, an object with a "default" key, or null. + */ +const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { + if (mode == null) return null; + if (typeof mode === "string") return mode; + if (Array.isArray(mode)) return mode[0] ?? null; + if (typeof mode === "object" && "default" in mode) { + const def = mode.default; + if (typeof def === "string") return def; + if (Array.isArray(def)) return def[0] ?? null; + } + return null; +}; + +const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { + const s = resolveMode(mode); + if (s == null || s === "") return "—"; return s.replace(/_/g, "-").toUpperCase(); }; @@ -302,9 +318,12 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { items.push({ type: "request", label: "Request received", offsetMs: 0 }); // Pre-call guardrails - const preCalls = sorted.filter((e) => e.guardrail_mode === "pre_call"); - const postCalls = sorted.filter((e) => e.guardrail_mode === "post_call" || e.guardrail_mode === "logging_only"); - const duringCalls = sorted.filter((e) => e.guardrail_mode === "during_call"); + const preCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "pre_call"); + const postCalls = sorted.filter((e) => { + const m = resolveMode(e.guardrail_mode); + return m === "post_call" || m === "logging_only"; + }); + const duringCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "during_call"); for (const e of preCalls) { const offsetMs = Math.round((e.end_time - baseTime) * 1000); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts index 3c78aacf85..fc487b04d7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts @@ -23,7 +23,7 @@ export interface GuardrailInformation { duration: number; end_time: number; start_time: number; - guardrail_mode: string; + guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse; From 20c1d984a610935368bacb215773135ac109cd5c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:18:42 -0700 Subject: [PATCH 15/57] [Test] UI: Add unit tests for 10 previously untested components Add Vitest + RTL tests covering DebugWarningBanner, HelpLink, ExportFormatSelector, ExportSummary, ExportTypeSelector, UsageExportHeader, MetricCard, ScoreChart, GuardrailConfig, and AgentHubTableColumns. 73 tests total. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../AIHub/AgentHubTableColumns.test.tsx | 132 ++++++++++++++++++ .../components/DebugWarningBanner.test.tsx | 38 +++++ .../ExportFormatSelector.test.tsx | 36 +++++ .../EntityUsageExport/ExportSummary.test.tsx | 35 +++++ .../ExportTypeSelector.test.tsx | 37 +++++ .../UsageExportHeader.test.tsx | 73 ++++++++++ .../GuardrailConfig.test.tsx | 84 +++++++++++ .../GuardrailsMonitor/MetricCard.test.tsx | 34 +++++ .../GuardrailsMonitor/ScoreChart.test.tsx | 47 +++++++ .../src/components/HelpLink.test.tsx | 128 +++++++++++++++++ 10 files changed, 644 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx create mode 100644 ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/HelpLink.test.tsx diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx new file mode 100644 index 0000000000..c32df2f546 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -0,0 +1,132 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { getAgentHubTableColumns, AgentHubData } from "./AgentHubTableColumns"; + +const mockAgent: AgentHubData = { + agent_id: "agent-1", + protocolVersion: "1.0", + name: "Test Agent", + description: "A test agent for unit testing", + url: "https://agent.example.com", + version: "2.0", + capabilities: { streaming: true, caching: false }, + defaultInputModes: ["text"], + defaultOutputModes: ["text", "image"], + skills: [ + { id: "s1", name: "Skill One", description: "First skill" }, + { id: "s2", name: "Skill Two", description: "Second skill" }, + { id: "s3", name: "Skill Three", description: "Third skill" }, + ], + is_public: true, +}; + +function TestTable({ data, publicPage = false }: { data: AgentHubData[]; publicPage?: boolean }) { + const showModal = vi.fn(); + const copyToClipboard = vi.fn(); + const columns = getAgentHubTableColumns(showModal, copyToClipboard, publicPage); + const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); + + return ( + + + {table.getHeaderGroups().map((hg) => ( + + {hg.headers.map((h) => ( + + ))} + + ))} + + + {table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
{flexRender(h.column.columnDef.header, h.getContext())}
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ ); +} + +describe("AgentHubTableColumns", () => { + it("should render", () => { + render(); + expect(screen.getByText("Test Agent")).toBeInTheDocument(); + }); + + it("should display the agent description", () => { + render(); + // Description appears in both the description column and the mobile view within agent name column + expect(screen.getAllByText("A test agent for unit testing").length).toBeGreaterThanOrEqual(1); + }); + + it("should display the version with a 'v' prefix", () => { + render(); + expect(screen.getByText("v2.0")).toBeInTheDocument(); + }); + + it("should display the protocol version", () => { + render(); + expect(screen.getByText("1.0")).toBeInTheDocument(); + }); + + it("should show skill count with correct pluralization", () => { + render(); + expect(screen.getByText("3 skills")).toBeInTheDocument(); + }); + + it("should show first two skills and '+1' for overflow", () => { + render(); + expect(screen.getByText("Skill One")).toBeInTheDocument(); + expect(screen.getByText("Skill Two")).toBeInTheDocument(); + expect(screen.getByText("+1")).toBeInTheDocument(); + }); + + it("should show only true capabilities as badges", () => { + render(); + expect(screen.getByText("streaming")).toBeInTheDocument(); + expect(screen.queryByText("caching")).not.toBeInTheDocument(); + }); + + it("should display I/O modes", () => { + render(); + expect(screen.getByText("text")).toBeInTheDocument(); + expect(screen.getByText("text, image")).toBeInTheDocument(); + }); + + it("should display 'Yes' badge for public agents", () => { + render(); + expect(screen.getByText("Yes")).toBeInTheDocument(); + }); + + it("should display 'No' badge for non-public agents", () => { + const privateAgent = { ...mockAgent, is_public: false }; + render(); + expect(screen.getByText("No")).toBeInTheDocument(); + }); + + it("should display a Details button", () => { + render(); + expect(screen.getByRole("button", { name: /details|info/i })).toBeInTheDocument(); + }); + + it("should show '-' when agent has no capabilities", () => { + const noCapAgent = { ...mockAgent, capabilities: {} }; + render(); + // The dash is rendered in the capabilities column + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("should show singular 'skill' for one skill", () => { + const oneSkillAgent = { + ...mockAgent, + skills: [{ id: "s1", name: "Only Skill", description: "One" }], + }; + render(); + expect(screen.getByText("1 skill")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx new file mode 100644 index 0000000000..4c99175162 --- /dev/null +++ b/ui/litellm-dashboard/src/components/DebugWarningBanner.test.tsx @@ -0,0 +1,38 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { vi } from "vitest"; +import { DebugWarningBanner } from "./DebugWarningBanner"; + +const mockUseHealthReadiness = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadiness", () => ({ + useHealthReadiness: () => mockUseHealthReadiness(), +})); + +describe("DebugWarningBanner", () => { + afterEach(() => { + vi.resetAllMocks(); + }); + + it("should render", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: true } }); + renderWithProviders(); + expect(screen.getByText(/Performance Warning/i)).toBeInTheDocument(); + }); + + it("should render nothing when debug mode is disabled", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: false } }); + const { container } = renderWithProviders(); + expect(container.firstChild).toBeNull(); + }); + + it("should render nothing when health data is undefined", () => { + mockUseHealthReadiness.mockReturnValue({ data: undefined }); + const { container } = renderWithProviders(); + expect(container.firstChild).toBeNull(); + }); + + it("should mention LITELLM_LOG=DEBUG in the description", () => { + mockUseHealthReadiness.mockReturnValue({ data: { is_detailed_debug: true } }); + renderWithProviders(); + expect(screen.getByText(/LITELLM_LOG=DEBUG/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx new file mode 100644 index 0000000000..a88f9cb116 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx @@ -0,0 +1,36 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import ExportFormatSelector from "./ExportFormatSelector"; + +describe("ExportFormatSelector", () => { + it("should render", () => { + render(); + expect(screen.getByText("Format")).toBeInTheDocument(); + }); + + it("should display the current value as csv", () => { + render(); + expect(screen.getByText("CSV (Excel, Google Sheets)")).toBeInTheDocument(); + }); + + it("should display the current value as json", () => { + render(); + expect(screen.getByText("JSON (includes metadata)")).toBeInTheDocument(); + }); + + it("should call onChange when a different format is selected", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + // Open the Ant Design Select dropdown + await user.click(screen.getByText("CSV (Excel, Google Sheets)")); + // Select JSON option from the dropdown + const jsonOption = await screen.findByText("JSON (includes metadata)", { + selector: ".ant-select-item-option-content", + }); + await user.click(jsonOption); + expect(onChange).toHaveBeenCalledWith("json", expect.anything()); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx new file mode 100644 index 0000000000..ab426e291b --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx @@ -0,0 +1,35 @@ +import { render, screen } from "@testing-library/react"; +import ExportSummary from "./ExportSummary"; + +describe("ExportSummary", () => { + const dateRange = { + from: new Date("2025-01-01"), + to: new Date("2025-01-31"), + }; + + it("should render", () => { + render(); + expect(screen.getByText(/2025/)).toBeInTheDocument(); + }); + + it("should display formatted date range", () => { + render(); + const text = screen.getByText(/\d+.*-.*\d+/); + expect(text).toBeInTheDocument(); + }); + + it("should show singular 'filter' for one filter", () => { + render(); + expect(screen.getByText(/1 filter$/)).toBeInTheDocument(); + }); + + it("should show plural 'filters' for multiple filters", () => { + render(); + expect(screen.getByText(/2 filters/)).toBeInTheDocument(); + }); + + it("should not show filter text when no filters applied", () => { + render(); + expect(screen.queryByText(/filter/)).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx new file mode 100644 index 0000000000..6ccf8822e0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx @@ -0,0 +1,37 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import ExportTypeSelector from "./ExportTypeSelector"; + +describe("ExportTypeSelector", () => { + it("should render", () => { + render(); + expect(screen.getByText("Export type")).toBeInTheDocument(); + }); + + it("should render all three radio options", () => { + render(); + expect(screen.getAllByRole("radio")).toHaveLength(3); + }); + + it("should interpolate entity type in labels", () => { + render(); + expect(screen.getByText(/Day-by-day breakdown by organization$/)).toBeInTheDocument(); + expect(screen.getByText(/organization and key/)).toBeInTheDocument(); + expect(screen.getByText(/organization and model/)).toBeInTheDocument(); + }); + + it("should call onChange when a different option is selected", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + await user.click(screen.getByText(/by team and key/)); + expect(onChange).toHaveBeenCalledWith("daily_with_keys"); + }); + + it("should have the correct radio checked based on value prop", () => { + render(); + const radios = screen.getAllByRole("radio"); + expect(radios[2]).toBeChecked(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx new file mode 100644 index 0000000000..729d6fd340 --- /dev/null +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx @@ -0,0 +1,73 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import UsageExportHeader from "./UsageExportHeader"; +import type { EntitySpendData } from "./types"; + +vi.mock("./EntityUsageExportModal", () => ({ + default: ({ isOpen, onClose }: { isOpen: boolean; onClose: () => void }) => + isOpen ? ( +
+ +
+ ) : null, +})); + +const defaultProps = { + dateValue: { from: new Date("2025-01-01"), to: new Date("2025-01-31") }, + entityType: "team" as const, + spendData: { + results: [], + metadata: { + total_spend: 0, + total_api_requests: 0, + total_successful_requests: 0, + total_failed_requests: 0, + total_tokens: 0, + }, + } satisfies EntitySpendData, +}; + +describe("UsageExportHeader", () => { + it("should render", () => { + renderWithProviders(); + expect(screen.getByRole("button", { name: /export data/i })).toBeInTheDocument(); + }); + + it("should open the export modal when the export button is clicked", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + expect(screen.getByTestId("export-modal")).toBeInTheDocument(); + }); + + it("should close the export modal when onClose is called", async () => { + const user = userEvent.setup(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /export data/i })); + await user.click(screen.getByRole("button", { name: /close/i })); + expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument(); + }); + + it("should not show filter dropdown when showFilters is false", () => { + renderWithProviders(); + expect(screen.queryByText(/filter/i)).not.toBeInTheDocument(); + }); + + it("should show filter dropdown when showFilters is true and options provided", () => { + renderWithProviders( + , + ); + expect(screen.getByText("Team")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx new file mode 100644 index 0000000000..fd9ad61599 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -0,0 +1,84 @@ +import { render, screen, act } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { GuardrailConfig } from "./GuardrailConfig"; + +describe("GuardrailConfig", () => { + const defaultProps = { + guardrailName: "Content Safety", + guardrailType: "Content Safety", + provider: "bedrock", + }; + + it("should render", () => { + render(); + expect(screen.getByText("Parameters")).toBeInTheDocument(); + }); + + it("should display the guardrail name in the parameters description", () => { + render(); + expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); + }); + + it("should show version history when 'View history' is clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /view history/i })); + expect(screen.getByText("Initial configuration")).toBeInTheDocument(); + expect(screen.getByText("Added custom categories list")).toBeInTheDocument(); + }); + + it("should toggle version history text between View/Hide", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /view history/i }); + await user.click(button); + expect(screen.getByRole("button", { name: /hide history/i })).toBeInTheDocument(); + }); + + it("should show custom code textarea when custom code override is toggled on", async () => { + const user = userEvent.setup(); + render(); + const switches = screen.getAllByRole("switch"); + // The second switch is the custom code override toggle + const customCodeSwitch = switches[1]; + await user.click(customCodeSwitch); + expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); + }); + + it("should hide custom code textarea when custom code override is off", () => { + render(); + // There's an input for categories, but no textarea + expect(screen.queryByPlaceholderText(/async def evaluate/)).not.toBeInTheDocument(); + }); + + it("should show the re-run button in idle state", () => { + render(); + expect(screen.getByRole("button", { name: /re-run on failing logs/i })).toBeInTheDocument(); + }); + + it("should show loading state when re-run is clicked", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); + vi.useRealTimers(); + }); + + it("should show success message after re-run completes", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); + render(); + await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); + act(() => { vi.advanceTimersByTime(2500); }); + expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); + vi.useRealTimers(); + }); + + it("should display the Revert and Save buttons", () => { + render(); + expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /save as v4/i })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx new file mode 100644 index 0000000000..9352cf4655 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.test.tsx @@ -0,0 +1,34 @@ +import { render, screen } from "@testing-library/react"; +import { MetricCard } from "./MetricCard"; + +describe("MetricCard", () => { + it("should render", () => { + render(); + expect(screen.getByText("Total Requests")).toBeInTheDocument(); + }); + + it("should display the numeric value", () => { + render(); + expect(screen.getByText("1234")).toBeInTheDocument(); + }); + + it("should display a string value", () => { + render(); + expect(screen.getByText("95.2%")).toBeInTheDocument(); + }); + + it("should display subtitle when provided", () => { + render(); + expect(screen.getByText("Last 7 days")).toBeInTheDocument(); + }); + + it("should not display subtitle when not provided", () => { + render(); + expect(screen.queryByText(/days/)).not.toBeInTheDocument(); + }); + + it("should display icon when provided", () => { + render(!} />); + expect(screen.getByTestId("icon")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx new file mode 100644 index 0000000000..b950dd9a30 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx @@ -0,0 +1,47 @@ +import { render, screen } from "@testing-library/react"; +import { ScoreChart } from "./ScoreChart"; + +vi.mock("@tremor/react", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + BarChart: ({ data, categories }: { data: unknown[]; categories: string[] }) => ( +
+ {data.length} data points +
+ ), + }; +}); + +describe("ScoreChart", () => { + it("should render", () => { + render(); + expect(screen.getByText("Request Outcomes Over Time")).toBeInTheDocument(); + }); + + it("should show empty state when no data provided", () => { + render(); + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should show empty state when data is an empty array", () => { + render(); + expect(screen.getByText("No chart data for this period")).toBeInTheDocument(); + }); + + it("should render chart when data is provided", () => { + const data = [ + { date: "2025-01-01", passed: 100, blocked: 5 }, + { date: "2025-01-02", passed: 120, blocked: 3 }, + ]; + render(); + expect(screen.getByTestId("bar-chart")).toBeInTheDocument(); + expect(screen.getByText("2 data points")).toBeInTheDocument(); + }); + + it("should pass correct categories to the chart", () => { + const data = [{ date: "2025-01-01", passed: 100, blocked: 5 }]; + render(); + expect(screen.getByTestId("bar-chart")).toHaveAttribute("data-categories", "passed,blocked"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/HelpLink.test.tsx b/ui/litellm-dashboard/src/components/HelpLink.test.tsx new file mode 100644 index 0000000000..2e9b44a9e9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/HelpLink.test.tsx @@ -0,0 +1,128 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HelpLink, HelpIcon, DocsMenu } from "./HelpLink"; + +describe("HelpLink", () => { + it("should render", () => { + render(); + expect(screen.getByRole("link")).toBeInTheDocument(); + }); + + it("should display default 'Learn more' text when no children provided", () => { + render(); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should display custom children text", () => { + render(Custom docs link); + expect(screen.getByText("Custom docs link")).toBeInTheDocument(); + }); + + it("should open in a new tab with noopener noreferrer", () => { + render(); + const link = screen.getByRole("link"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("should have accessible screen reader text", () => { + render(); + expect(screen.getByText("(opens in a new tab)")).toBeInTheDocument(); + }); +}); + +describe("HelpIcon", () => { + it("should render", () => { + render(); + expect(screen.getByRole("button", { name: /help information/i })).toBeInTheDocument(); + }); + + it("should show tooltip content on mouse enter", async () => { + const user = userEvent.setup(); + render(); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Helpful tooltip text")).toBeInTheDocument(); + }); + + it("should hide tooltip content on mouse leave", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /help information/i }); + await user.hover(button); + await user.unhover(button); + expect(screen.queryByText("Helpful tooltip text")).not.toBeInTheDocument(); + }); + + it("should show learn more link when learnMoreHref is provided", async () => { + const user = userEvent.setup(); + render(); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Learn more")).toBeInTheDocument(); + }); + + it("should use custom learnMoreText when provided", async () => { + const user = userEvent.setup(); + render( + , + ); + await user.hover(screen.getByRole("button", { name: /help information/i })); + expect(screen.getByText("Read docs")).toBeInTheDocument(); + }); +}); + +describe("DocsMenu", () => { + const items = [ + { label: "Custom pricing", href: "https://docs.example.com/pricing" }, + { label: "Spend tracking", href: "https://docs.example.com/spend" }, + ]; + + it("should render", () => { + render(); + expect(screen.getByRole("button", { name: /docs/i })).toBeInTheDocument(); + }); + + it("should show menu items when clicked", async () => { + const user = userEvent.setup(); + render(); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + expect(screen.getByText("Spend tracking")).toBeInTheDocument(); + }); + + it("should hide menu items when clicked again", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /docs/i }); + await user.click(button); + await user.click(button); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("should set aria-expanded correctly", async () => { + const user = userEvent.setup(); + render(); + const button = screen.getByRole("button", { name: /docs/i }); + expect(button).toHaveAttribute("aria-expanded", "false"); + await user.click(button); + expect(button).toHaveAttribute("aria-expanded", "true"); + }); + + it("should close menu when clicking outside", async () => { + const user = userEvent.setup(); + render( +
+ + +
, + ); + await user.click(screen.getByRole("button", { name: /docs/i })); + expect(screen.getByText("Custom pricing")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: /outside/i })); + expect(screen.queryByText("Custom pricing")).not.toBeInTheDocument(); + }); + + it("should display custom children text", () => { + render(Help); + expect(screen.getByText("Help")).toBeInTheDocument(); + }); +}); From 1c8b5f77c96be048e83a2b35c7679fd8d546e53b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:29:57 -0700 Subject: [PATCH 16/57] address greptile review feedback (greploop iteration 1) Add modeMatches() helper so array guardrail_mode values (e.g. ["pre_call", "post_call"]) place the entry in all matching timeline buckets, not just the first. Updated test to verify both buckets. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../GuardrailViewer/GuardrailViewer.test.tsx | 6 ++- .../GuardrailViewer/GuardrailViewer.tsx | 37 ++++++++++++++----- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 60778a1337..b5e04c7244 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -170,14 +170,18 @@ describe("GuardrailViewer", () => { expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); }); - it("renders without crashing when guardrail_mode is an array", () => { + it("renders without crashing when guardrail_mode is an array and shows in both timeline buckets", () => { const data = makeGuardrailInformation({ guardrail_mode: ["pre_call", "post_call"], }); renderWithProviders(); expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + // Mode badge shows first element formatted expect(screen.getByText("PRE-CALL")).toBeInTheDocument(); + // Entry should appear in both pre-call and post-call timeline sections + expect(screen.getByText(/Pre-call guardrail:/)).toBeInTheDocument(); + expect(screen.getByText(/Post-call guardrail:/)).toBeInTheDocument(); }); it("integration: renders with real Bedrock details without mocks", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 0c37b70c8b..1056076fcd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -78,8 +78,8 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ ]); /** - * Extracts a plain string from guardrail_mode, which may be a string, - * an array of strings, an object with a "default" key, or null. + * Extracts a plain string from guardrail_mode for display purposes. + * Returns the first mode when multiple are present. */ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { if (mode == null) return null; @@ -93,6 +93,25 @@ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | nul return null; }; +/** + * Checks whether guardrail_mode includes the given target stage. + * Handles arrays (multi-stage guardrails) by checking all elements. + */ +const modeMatches = ( + mode: GuardrailInformation["guardrail_mode"], + target: string, +): boolean => { + if (mode == null) return false; + if (typeof mode === "string") return mode === target; + if (Array.isArray(mode)) return mode.includes(target); + if (typeof mode === "object" && "default" in mode) { + const def = mode.default; + if (typeof def === "string") return def === target; + if (Array.isArray(def)) return (def as string[]).includes(target); + } + return false; +}; + const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { const s = resolveMode(mode); if (s == null || s === "") return "—"; @@ -317,13 +336,13 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { // Request received items.push({ type: "request", label: "Request received", offsetMs: 0 }); - // Pre-call guardrails - const preCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "pre_call"); - const postCalls = sorted.filter((e) => { - const m = resolveMode(e.guardrail_mode); - return m === "post_call" || m === "logging_only"; - }); - const duringCalls = sorted.filter((e) => resolveMode(e.guardrail_mode) === "during_call"); + // Pre-call guardrails — use modeMatches so array modes (e.g. ["pre_call", "post_call"]) + // place the entry in every matching bucket. + const preCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "pre_call")); + const postCalls = sorted.filter( + (e) => modeMatches(e.guardrail_mode, "post_call") || modeMatches(e.guardrail_mode, "logging_only"), + ); + const duringCalls = sorted.filter((e) => modeMatches(e.guardrail_mode, "during_call")); for (const e of preCalls) { const offsetMs = Math.round((e.end_time - baseTime) * 1000); From 40721ab18f15bf4e3df2c1c04d8aba2be155e9e0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:31:25 -0700 Subject: [PATCH 17/57] address greptile review feedback (greploop iteration 1) - Move vi.useRealTimers() to afterEach for proper cleanup - Use label-based DOM queries instead of fragile positional indexes - Remove leftover debug console.log from AgentHubTableColumns.tsx Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/AIHub/AgentHubTableColumns.tsx | 1 - .../ExportTypeSelector.test.tsx | 4 ++-- .../GuardrailConfig.test.tsx | 20 +++++++++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx index c6a8c0b9da..09b1c14761 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx @@ -194,7 +194,6 @@ export const getAgentHubTableColumns = ( return publicA - publicB; }, cell: ({ row }) => { - console.log(`CHECKPOINT 1: ${JSON.stringify(row.original)}`); const agent = row.original; return agent.is_public === true ? ( diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx index 6ccf8822e0..d9469f095a 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportTypeSelector.test.tsx @@ -31,7 +31,7 @@ describe("ExportTypeSelector", () => { it("should have the correct radio checked based on value prop", () => { render(); - const radios = screen.getAllByRole("radio"); - expect(radios[2]).toBeChecked(); + const modelRadio = screen.getByRole("radio", { name: /by team and model/i }); + expect(modelRadio).toBeChecked(); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index fd9ad61599..38f6568198 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -10,6 +10,10 @@ describe("GuardrailConfig", () => { provider: "bedrock", }; + afterEach(() => { + vi.useRealTimers(); + }); + it("should render", () => { render(); expect(screen.getByText("Parameters")).toBeInTheDocument(); @@ -39,10 +43,16 @@ describe("GuardrailConfig", () => { it("should show custom code textarea when custom code override is toggled on", async () => { const user = userEvent.setup(); render(); - const switches = screen.getAllByRole("switch"); - // The second switch is the custom code override toggle - const customCodeSwitch = switches[1]; - await user.click(customCodeSwitch); + // Walk up from "Custom Code Override" heading to find the enclosing section, + // then locate the switch within it + const heading = screen.getByText("Custom Code Override"); + let container = heading.parentElement; + let customCodeSwitch: Element | null = null; + while (container && !customCodeSwitch) { + customCodeSwitch = container.querySelector('[role="switch"]'); + container = container.parentElement; + } + await user.click(customCodeSwitch!); expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); }); @@ -63,7 +73,6 @@ describe("GuardrailConfig", () => { render(); await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); - vi.useRealTimers(); }); it("should show success message after re-run completes", async () => { @@ -73,7 +82,6 @@ describe("GuardrailConfig", () => { await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); act(() => { vi.advanceTimersByTime(2500); }); expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); - vi.useRealTimers(); }); it("should display the Revert and Save buttons", () => { From 6902355f5ba7ae6de8af2366d8ec0ce372c20b9b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:34:03 -0700 Subject: [PATCH 18/57] address greptile review feedback (greploop iteration 2) Replace unsafe `as string[]` cast in modeMatches with runtime type check via `.some()`. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/view_logs/GuardrailViewer/GuardrailViewer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1056076fcd..28247d4b2a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -107,7 +107,7 @@ const modeMatches = ( if (typeof mode === "object" && "default" in mode) { const def = mode.default; if (typeof def === "string") return def === target; - if (Array.isArray(def)) return (def as string[]).includes(target); + if (Array.isArray(def)) return def.some((x) => typeof x === "string" && x === target); } return false; }; From 7714d1be0b236871ffd24424d8513fd4630eaffb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:40:56 -0700 Subject: [PATCH 19/57] address greptile review feedback (greploop iteration 3) Add typeof string guards to all array element returns in resolveMode to prevent non-string values from sneaking through via any-widening. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../view_logs/GuardrailViewer/GuardrailViewer.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 28247d4b2a..5608e3677f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -84,11 +84,17 @@ const PROVIDERS_WITH_CUSTOM_RENDERERS = new Set([ const resolveMode = (mode: GuardrailInformation["guardrail_mode"]): string | null => { if (mode == null) return null; if (typeof mode === "string") return mode; - if (Array.isArray(mode)) return mode[0] ?? null; + if (Array.isArray(mode)) { + const first = mode[0]; + return typeof first === "string" ? first : null; + } if (typeof mode === "object" && "default" in mode) { const def = mode.default; if (typeof def === "string") return def; - if (Array.isArray(def)) return def[0] ?? null; + if (Array.isArray(def)) { + const first = def[0]; + return typeof first === "string" ? first : null; + } } return null; }; From 56ed8379e297e20fb219817690d3b197c440ab69 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:42:19 -0700 Subject: [PATCH 20/57] address greptile review feedback (greploop iteration 2) - Add explicit vi import to ScoreChart.test.tsx - Use custom matcher for I/O modes to avoid cross-element text issues - Use version-agnostic regex for Save button assertion - Add comments noting placeholder data in GuardrailConfig tests Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/AIHub/AgentHubTableColumns.test.tsx | 10 ++++++++-- .../GuardrailsMonitor/GuardrailConfig.test.tsx | 5 ++++- .../components/GuardrailsMonitor/ScoreChart.test.tsx | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx index c32df2f546..703a24d8d6 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -94,8 +94,14 @@ describe("AgentHubTableColumns", () => { it("should display I/O modes", () => { render(); - expect(screen.getByText("text")).toBeInTheDocument(); - expect(screen.getByText("text, image")).toBeInTheDocument(); + // "In:" and "Out:" are in children; getByText with exact:false + // matches against the element's full textContent across child nodes + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "In: text" + )).toBeInTheDocument(); + expect(screen.getByText((_, el) => + el?.tagName === "P" && el.textContent === "Out: text, image" + )).toBeInTheDocument(); }); it("should display 'Yes' badge for public agents", () => { diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index 38f6568198..ac84c270d8 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -24,6 +24,8 @@ describe("GuardrailConfig", () => { expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); }); + // Note: Version history entries are hardcoded placeholders in the component. + // These assertions will need updating when wired to real API data. it("should show version history when 'View history' is clicked", async () => { const user = userEvent.setup(); render(); @@ -87,6 +89,7 @@ describe("GuardrailConfig", () => { it("should display the Revert and Save buttons", () => { render(); expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /save as v4/i })).toBeInTheDocument(); + // The component's hardcoded default version is "v3", so Save shows "v4" + expect(screen.getByRole("button", { name: /save as v\d+/i })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx index b950dd9a30..c32e674578 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/ScoreChart.test.tsx @@ -1,4 +1,5 @@ import { render, screen } from "@testing-library/react"; +import { vi } from "vitest"; import { ScoreChart } from "./ScoreChart"; vi.mock("@tremor/react", async (importOriginal) => { From bbc120095e2dcc9477f76d647d01490699861114 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 12:48:01 -0700 Subject: [PATCH 21/57] address greptile review feedback (greploop iteration 3) - Remove Ant Design CSS class selector coupling in ExportFormatSelector test - Lift mock fns out of TestTable component body to enable callback assertions Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/AIHub/AgentHubTableColumns.test.tsx | 14 +++++++++++--- .../ExportFormatSelector.test.tsx | 4 +--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx index 703a24d8d6..083e67c297 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.test.tsx @@ -22,9 +22,17 @@ const mockAgent: AgentHubData = { is_public: true, }; -function TestTable({ data, publicPage = false }: { data: AgentHubData[]; publicPage?: boolean }) { - const showModal = vi.fn(); - const copyToClipboard = vi.fn(); +function TestTable({ + data, + publicPage = false, + showModal = vi.fn(), + copyToClipboard = vi.fn(), +}: { + data: AgentHubData[]; + publicPage?: boolean; + showModal?: ReturnType; + copyToClipboard?: ReturnType; +}) { const columns = getAgentHubTableColumns(showModal, copyToClipboard, publicPage); const table = useReactTable({ data, columns, getCoreRowModel: getCoreRowModel() }); diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx index a88f9cb116..d20d24992f 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportFormatSelector.test.tsx @@ -27,9 +27,7 @@ describe("ExportFormatSelector", () => { // Open the Ant Design Select dropdown await user.click(screen.getByText("CSV (Excel, Google Sheets)")); // Select JSON option from the dropdown - const jsonOption = await screen.findByText("JSON (includes metadata)", { - selector: ".ant-select-item-option-content", - }); + const jsonOption = await screen.findByText("JSON (includes metadata)"); await user.click(jsonOption); expect(onChange).toHaveBeenCalledWith("json", expect.anything()); }); From fcea5606827552b506fb1de478ec1c6d095e7152 Mon Sep 17 00:00:00 2001 From: Avik Kumar Date: Wed, 18 Mar 2026 15:58:13 -0400 Subject: [PATCH 22/57] fix(langsmith): populate usage_metadata in outputs for Cost column LangSmith reads the Cost column from outputs.usage_metadata.total_cost, but LangsmithLogger._prepare_log_data never wrote to that key. The response_cost was already computed in StandardLoggingPayload but was not forwarded to the outputs dict. Inject usage_metadata with input_tokens, output_tokens, total_tokens, and total_cost into the outputs dict so LangSmith can display cost. Fixes #24001 Made-with: Cursor --- litellm/integrations/langsmith.py | 14 ++++- .../integrations/test_langsmith_init.py | 54 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 03845af521..ef2d30bb26 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -153,11 +153,23 @@ class LangsmithLogger(CustomBatchLogger): if key in requester_metadata and key not in extra_metadata: extra_metadata[key] = requester_metadata[key] + outputs = payload["response"] + if isinstance(outputs, dict): + outputs = {**outputs} + else: + outputs = {"output": outputs} + outputs["usage_metadata"] = { + "input_tokens": payload.get("prompt_tokens", 0), + "output_tokens": payload.get("completion_tokens", 0), + "total_tokens": payload.get("total_tokens", 0), + "total_cost": payload.get("response_cost", 0), + } + data = { "name": run_name, "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" "inputs": payload, - "outputs": payload["response"], + "outputs": outputs, "session_name": project_name, "start_time": payload["startTime"], "end_time": payload["endTime"], diff --git a/tests/test_litellm/integrations/test_langsmith_init.py b/tests/test_litellm/integrations/test_langsmith_init.py index 9f7db4095b..779e7b4c94 100644 --- a/tests/test_litellm/integrations/test_langsmith_init.py +++ b/tests/test_litellm/integrations/test_langsmith_init.py @@ -132,3 +132,57 @@ class TestLangsmithLoggerInit: assert ( logger.sampling_rate >= 0.0 ), f"sampling_rate should be non-negative, got {logger.sampling_rate}" + + +class TestLangsmithPrepareLogData: + """Regression test for #24001: _prepare_log_data must inject + usage_metadata into outputs so LangSmith's Cost column is populated.""" + + @patch("asyncio.create_task") + @patch.dict(os.environ, {"LANGSMITH_SAMPLING_RATE": "1"}, clear=False) + def test_outputs_contain_usage_metadata(self, mock_create_task): + logger = LangsmithLogger( + langsmith_api_key="test-key", + langsmith_project="test-project", + ) + + payload = { + "id": "test-id", + "response": {"choices": [{"message": {"content": "hi"}}]}, + "metadata": {}, + "startTime": 1.0, + "endTime": 2.0, + "request_tags": [], + "error_str": None, + "status": "success", + "response_cost": 0.0042, + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + } + + kwargs = { + "litellm_params": {"metadata": {}}, + "standard_logging_object": payload, + } + + credentials = { + "LANGSMITH_API_KEY": "test-key", + "LANGSMITH_PROJECT": "test-project", + "LANGSMITH_BASE_URL": "https://api.smith.langchain.com", + } + + data = logger._prepare_log_data( + kwargs=kwargs, + response_obj=None, + start_time=1.0, + end_time=2.0, + credentials=credentials, + ) + + assert "usage_metadata" in data["outputs"] + um = data["outputs"]["usage_metadata"] + assert um["total_cost"] == 0.0042 + assert um["input_tokens"] == 100 + assert um["output_tokens"] == 50 + assert um["total_tokens"] == 150 From eb7efa36daf03f23d81d137122c0b8e3f1575067 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 13:05:53 -0700 Subject: [PATCH 23/57] Add missing permission options to PERMISSION_OPTIONS list Adds /key/info, /key/list, /key/aliases, and /team/daily/activity to the hardcoded PERMISSION_OPTIONS in TeamSSOSettings.tsx. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/components/TeamSSOSettings.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx index a9c07cdbcf..98745d388a 100644 --- a/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/TeamSSOSettings.tsx @@ -26,6 +26,10 @@ const PERMISSION_OPTIONS = [ "/key/unblock", "/key/bulk_update", "/key/{key_id}/reset_spend", + "/key/info", + "/key/list", + "/key/aliases", + "/team/daily/activity", ]; interface SettingRowProps { From 51f78b7d7250a86f127d02fbe964968b2a8a41e2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 13:09:17 -0700 Subject: [PATCH 24/57] address greptile review feedback (greploop iteration 4) - Add guard assertion before non-null click on custom code switch - Use await act(async ...) for timer advancement to avoid act warnings - Pin locale in date range assertion for CI determinism Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/EntityUsageExport/ExportSummary.test.tsx | 6 ++++-- .../components/GuardrailsMonitor/GuardrailConfig.test.tsx | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx index ab426e291b..1aeee42f74 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/ExportSummary.test.tsx @@ -14,8 +14,10 @@ describe("ExportSummary", () => { it("should display formatted date range", () => { render(); - const text = screen.getByText(/\d+.*-.*\d+/); - expect(text).toBeInTheDocument(); + // Pin locale to en-US so test is deterministic regardless of CI runner locale + const expectedFrom = dateRange.from!.toLocaleDateString("en-US"); + const expectedTo = dateRange.to!.toLocaleDateString("en-US"); + expect(screen.getByText(`${expectedFrom} - ${expectedTo}`)).toBeInTheDocument(); }); it("should show singular 'filter' for one filter", () => { diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx index ac84c270d8..54c7ebabe7 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/GuardrailConfig.test.tsx @@ -54,7 +54,10 @@ describe("GuardrailConfig", () => { customCodeSwitch = container.querySelector('[role="switch"]'); container = container.parentElement; } - await user.click(customCodeSwitch!); + if (!customCodeSwitch) { + throw new Error("Could not find the Custom Code Override switch via DOM traversal"); + } + await user.click(customCodeSwitch); expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); }); @@ -82,7 +85,7 @@ describe("GuardrailConfig", () => { const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); render(); await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); - act(() => { vi.advanceTimersByTime(2500); }); + await act(async () => { vi.advanceTimersByTime(2500); }); expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); }); From 98311e0f0a57094eb8cba5b0ec82e5af671b890d Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 18 Mar 2026 15:07:50 -0500 Subject: [PATCH 25/57] Preserve router model_group in generic API logs --- litellm/router.py | 8 ++++ .../test_router_helper_utils.py | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index f34368172a..d2acd4c1f5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3874,10 +3874,18 @@ class Router: The response from the handler function """ handler_name = original_function.__name__ + metadata_variable_name = _get_router_metadata_variable_name( + function_name="generic_api_call" + ) try: verbose_router_logger.debug( f"Inside _generic_api_call() - handler: {handler_name}, model: {model}; kwargs: {kwargs}" ) + self._update_kwargs_before_fallbacks( + model=model, + kwargs=kwargs, + metadata_variable_name=metadata_variable_name, + ) deployment = self.get_available_deployment( model=model, messages=kwargs.get("messages", None), diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 3001aec8b8..75c5250b3d 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1568,6 +1568,43 @@ def test_handle_clientside_credential_with_deployment_model_name(model_list): print("✓ _handle_clientside_credential test passed!") +def test_sync_generic_api_call_preserves_requested_model_group_in_logs(): + router = Router( + model_list=[ + { + "model_name": "claude-sonnet-4-6", + "litellm_params": { + "model": "bedrock/global.anthropic.claude-sonnet-4-6", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-west-2", + }, + } + ] + ) + + captured_kwargs = {} + + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} + + response = router._generic_api_call_with_fallbacks( + model="claude-sonnet-4-6", + original_function=mock_original_function, + ) + + assert response == {"status": "ok"} + assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" + assert ( + captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["deployment"] + == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + + @pytest.mark.parametrize( "function_name, expected_metadata_key", [ From 845ad042913dd3d85d7f5d484fc0acc9ea990f14 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 18 Mar 2026 15:25:44 -0500 Subject: [PATCH 26/57] Address router generic API review feedback --- litellm/router.py | 1 + .../test_router_helper_utils.py | 80 +++++++++++++++---- 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d2acd4c1f5..5263e966ab 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3890,6 +3890,7 @@ class Router: model=model, messages=kwargs.get("messages", None), specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, ) self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, function_name="generic_api_call" diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 75c5250b3d..34a19f5ce7 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1583,26 +1583,74 @@ def test_sync_generic_api_call_preserves_requested_model_group_in_logs(): ] ) - captured_kwargs = {} + try: + captured_kwargs = {} - def mock_original_function(**kwargs): - captured_kwargs.update(kwargs) - return {"status": "ok"} + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} - response = router._generic_api_call_with_fallbacks( - model="claude-sonnet-4-6", - original_function=mock_original_function, + response = router._generic_api_call_with_fallbacks( + model="claude-sonnet-4-6", + original_function=mock_original_function, + ) + + assert response == {"status": "ok"} + assert ( + captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" + ) + assert ( + captured_kwargs["litellm_metadata"]["deployment"] + == "bedrock/global.anthropic.claude-sonnet-4-6" + ) + finally: + router.discard() + + +def test_sync_generic_api_call_uses_request_kwargs_for_deployment_selection(): + router = Router( + model_list=[ + { + "model_name": "regional-model", + "litellm_params": { + "model": "anthropic/us-model", + "api_key": "test-api-key", + "region_name": "us", + }, + }, + { + "model_name": "regional-model", + "litellm_params": { + "model": "anthropic/eu-model", + "api_key": "test-api-key", + "region_name": "eu", + }, + }, + ], + enable_pre_call_checks=True, ) - assert response == {"status": "ok"} - assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" - assert ( - captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" - ) - assert ( - captured_kwargs["litellm_metadata"]["deployment"] - == "bedrock/global.anthropic.claude-sonnet-4-6" - ) + try: + captured_kwargs = {} + + def mock_original_function(**kwargs): + captured_kwargs.update(kwargs) + return {"status": "ok"} + + response = router._generic_api_call_with_fallbacks( + model="regional-model", + original_function=mock_original_function, + messages=[{"role": "user", "content": "Hello from Europe"}], + allowed_model_region="eu", + ) + + assert response == {"status": "ok"} + assert captured_kwargs["model"] == "anthropic/eu-model" + finally: + router.discard() @pytest.mark.parametrize( From b00096f2a08a5d56d2a1d84c663b833dcdda5f66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Mar 2026 20:33:09 +0000 Subject: [PATCH 27/57] chore: regenerate poetry.lock to match pyproject.toml (#2) Co-authored-by: github-actions[bot] --- poetry.lock | 73 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/poetry.lock b/poetry.lock index 591d0c270e..be87e51f80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "a2a-sdk" @@ -7,11 +7,11 @@ description = "A2A Python SDK" optional = false python-versions = ">=3.10" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.10\"" files = [ {file = "a2a_sdk-0.3.22-py3-none-any.whl", hash = "sha256:b98701135bb90b0ff85d35f31533b6b7a299bf810658c1c65f3814a6c15ea385"}, {file = "a2a_sdk-0.3.22.tar.gz", hash = "sha256:77a5694bfc4f26679c11b70c7f1062522206d430b34bc1215cfbb1eba67b7e7d"}, ] +markers = {main = "python_version >= \"3.10\" and extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] google-api-core = ">=1.26.0" @@ -385,6 +385,7 @@ files = [ {file = "azure_core-1.36.0-py3-none-any.whl", hash = "sha256:fee9923a3a753e94a259563429f3644aaf05c486d45b1215d098115102d91d3b"}, {file = "azure_core-1.36.0.tar.gz", hash = "sha256:22e5605e6d0bf1d229726af56d9e92bc37b6e726b141a18be0b4d424131741b7"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] requests = ">=2.21.0" @@ -405,6 +406,7 @@ files = [ {file = "azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651"}, {file = "azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] azure-core = ">=1.31.0" @@ -598,7 +600,7 @@ files = [ {file = "cachetools-6.2.2-py3-none-any.whl", hash = "sha256:6c09c98183bf58560c97b2abfcedcbaf6a896a490f534b031b661d3723b45ace"}, {file = "cachetools-6.2.2.tar.gz", hash = "sha256:8e6d266b25e539df852251cfd6f990b4bc3a141db73b939058d809ebd2590fc6"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "certifi" @@ -705,7 +707,7 @@ files = [ {file = "cffi-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:b882b3df248017dba09d6b16defe9b5c407fe32fc7c65a9c69798e6175601be9"}, {file = "cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529"}, ] -markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "(platform_python_implementation != \"PyPy\" or extra == \"proxy\") and (python_version >= \"3.10\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\")", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} @@ -1055,6 +1057,7 @@ files = [ {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] +markers = {main = "python_version >= \"3.10\" and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\") or extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} @@ -1837,11 +1840,11 @@ description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.14\"" files = [ {file = "google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7"}, {file = "google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300"}, ] +markers = {main = "python_version >= \"3.14\" and (extra == \"extra-proxy\" or extra == \"google\")", proxy-dev = "python_version >= \"3.14\""} [package.dependencies] google-auth = ">=2.14.1,<3.0.0" @@ -1869,7 +1872,7 @@ files = [ {file = "google_api_core-2.28.1-py3-none-any.whl", hash = "sha256:4021b0f8ceb77a6fb4de6fde4502cecab45062e66ff4f2895169e0b35bc9466c"}, {file = "google_api_core-2.28.1.tar.gz", hash = "sha256:2b405df02d68e68ce0fbc138559e6036559e685159d148ae5861013dc201baf8"}, ] -markers = {main = "(python_version >= \"3.10\" or extra == \"google\" or extra == \"extra-proxy\") and python_version < \"3.14\"", proxy-dev = "python_version >= \"3.10\" and python_version < \"3.14\""} +markers = {main = "python_version < \"3.14\" and (extra == \"extra-proxy\" or extra == \"google\")", proxy-dev = "python_version >= \"3.10\" and python_version < \"3.14\""} [package.dependencies] google-auth = ">=2.14.1,<3.0.0" @@ -1906,7 +1909,7 @@ files = [ {file = "google_auth-2.43.0-py2.py3-none-any.whl", hash = "sha256:af628ba6fa493f75c7e9dbe9373d148ca9f4399b5ea29976519e0a3848eddd16"}, {file = "google_auth-2.43.0.tar.gz", hash = "sha256:88228eee5fc21b62a1b5fe773ca15e67778cb07dc8363adcb4a8827b52d81483"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] cachetools = ">=2.0.0,<7.0" @@ -2078,11 +2081,11 @@ files = [ ] [package.dependencies] -google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} -google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0dev" -grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" -proto-plus = ">=1.22.3,<2.0.0dev" -protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" +google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.11.dev0,<3.0.0.dev0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0" +grpc-google-iam-v1 = ">=0.12.4,<1.0.0.dev0" +proto-plus = ">=1.22.3,<2.0.0.dev0" +protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" [[package]] name = "google-cloud-resource-manager" @@ -2264,7 +2267,7 @@ files = [ {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\") or extra == \"google\" or extra == \"extra-proxy\""} [package.dependencies] grpcio = {version = ">=1.44.0,<2.0.0", optional = true, markers = "extra == \"grpc\""} @@ -2673,11 +2676,11 @@ description = "Consume Server-Sent Event (SSE) messages with HTTPX." optional = false python-versions = ">=3.9" groups = ["main", "proxy-dev"] -markers = "python_version >= \"3.10\"" files = [ {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, ] +markers = {main = "python_version >= \"3.10\" and (extra == \"proxy\" or extra == \"extra-proxy\")", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "huey" @@ -3042,7 +3045,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.7.1" @@ -3219,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.56" +version = "0.4.57" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.56-py3-none-any.whl", hash = "sha256:52dbe3b5358c790e77e12f1ec5ef8e7508b383c2aaf41299750b6fb400908ee7"}, - {file = "litellm_proxy_extras-0.4.56.tar.gz", hash = "sha256:63ad59baa0defccc5c929cfd933ee7e32a6614b0fc5fa0fc45a12d7608e33f08"}, + {file = "litellm_proxy_extras-0.4.57-py3-none-any.whl", hash = "sha256:04538223cd80318a72d70c6e10f701598e58c763368296a6503c674c92fbdb62"}, + {file = "litellm_proxy_extras-0.4.57.tar.gz", hash = "sha256:ef9b95dc42237614216833bd5d46ebf9dea1caa5ea14ea1a66d7f7842b224ec2"}, ] [[package]] @@ -3713,6 +3716,7 @@ files = [ {file = "msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1"}, {file = "msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cryptography = ">=2.5,<49" @@ -3733,6 +3737,7 @@ files = [ {file = "msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca"}, {file = "msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4"}, ] +markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] msal = ">=1.29,<2" @@ -3983,6 +3988,7 @@ files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, ] +markers = {main = "extra == \"extra-proxy\""} [[package]] name = "numpy" @@ -4105,7 +4111,7 @@ files = [ {file = "opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950"}, {file = "opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] importlib-metadata = ">=6.0,<8.8.0" @@ -4220,7 +4226,7 @@ files = [ {file = "opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c"}, {file = "opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] opentelemetry-api = "1.39.1" @@ -4238,7 +4244,7 @@ files = [ {file = "opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb"}, {file = "opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953"}, ] -markers = {main = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and extra == \"mlflow\""} [package.dependencies] opentelemetry-api = "1.39.1" @@ -4722,6 +4728,7 @@ files = [ {file = "prisma-0.11.0-py3-none-any.whl", hash = "sha256:22bb869e59a2968b99f3483bb417717273ffbc569fd1e9ceed95e5614cbaf53a"}, {file = "prisma-0.11.0.tar.gz", hash = "sha256:3f2f2fd2361e1ec5ff655f2a04c7860c2f2a5bc4c91f78ca9c5c6349735bf693"}, ] +markers = {main = "extra == \"extra-proxy\""} [package.dependencies] click = ">=7.1.2" @@ -4895,7 +4902,7 @@ files = [ {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] protobuf = ">=3.19.0,<7.0.0" @@ -4923,7 +4930,7 @@ files = [ {file = "protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5"}, {file = "protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\""} [[package]] name = "psutil" @@ -5083,7 +5090,7 @@ files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [[package]] name = "pyasn1-modules" @@ -5096,7 +5103,7 @@ files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] pyasn1 = ">=0.6.1,<0.7.0" @@ -5124,7 +5131,7 @@ files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] -markers = {main = "implementation_name != \"PyPy\" and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} +markers = {main = "implementation_name != \"PyPy\" and (platform_python_implementation != \"PyPy\" or extra == \"proxy\") and (python_version >= \"3.10\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"proxy\" or extra == \"extra-proxy\" or extra == \"mlflow\")", dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\""} [[package]] name = "pydantic" @@ -5347,6 +5354,7 @@ files = [ {file = "pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c"}, {file = "pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b"}, ] +markers = {main = "(python_version <= \"3.13\" or extra == \"proxy\" or extra == \"extra-proxy\") and (extra == \"extra-proxy\" or extra == \"proxy\")"} [package.dependencies] cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} @@ -6290,7 +6298,7 @@ files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, ] -markers = {main = "extra == \"google\" or extra == \"extra-proxy\" or python_version >= \"3.10\"", proxy-dev = "python_version >= \"3.10\""} +markers = {main = "python_version >= \"3.10\" and (extra == \"extra-proxy\" or extra == \"google\" or extra == \"mlflow\") or extra == \"google\" or extra == \"extra-proxy\"", proxy-dev = "python_version >= \"3.10\""} [package.dependencies] pyasn1 = ">=0.1.3" @@ -6336,10 +6344,10 @@ files = [ ] [package.dependencies] -botocore = ">=1.37.4,<2.0a.0" +botocore = ">=1.37.4,<2.0a0" [package.extras] -crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a0)"] [[package]] name = "scikit-learn" @@ -6492,9 +6500,9 @@ tornado = ">=6.4.2,<7" urllib3 = ">=1.26,<3" [package.extras] -all = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)", "cohere (>=5.9.4,<6.00)", "dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\"", "google-cloud-aiplatform (>=1.45.0,<2)", "ipykernel (>=6.25.0,<7)", "llama-cpp-python (>=0.2.28,<0.2.86) ; python_version < \"3.13\"", "mistralai (>=0.0.12,<0.1.0)", "mypy (>=1.7.1,<2)", "ollama (>=0.1.7)", "pillow (>=10.2.0,<11.0.0) ; python_version < \"3.13\"", "pinecone[asyncio] (>=7.0.0,<8.0.0)", "psycopg[binary] (>=3.1.0,<4)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "qdrant-client (>=1.11.1,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "sentence-transformers (>=5.0.0) ; python_version < \"3.13\"", "tokenizers (>=0.19) ; python_version < \"3.13\"", "torch (>=2.6.0) ; python_version < \"3.13\"", "torchvision (>=0.17.0) ; python_version < \"3.13\"", "transformers (>=4.36.2) ; python_version < \"3.13\"", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] +all = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)", "cohere (>=5.9.4,<6.0)", "dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\"", "google-cloud-aiplatform (>=1.45.0,<2)", "ipykernel (>=6.25.0,<7)", "llama-cpp-python (>=0.2.28,<0.2.86) ; python_version < \"3.13\"", "mistralai (>=0.0.12,<0.1.0)", "mypy (>=1.7.1,<2)", "ollama (>=0.1.7)", "pillow (>=10.2.0,<11.0.0) ; python_version < \"3.13\"", "pinecone[asyncio] (>=7.0.0,<8.0.0)", "psycopg[binary] (>=3.1.0,<4)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "qdrant-client (>=1.11.1,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "sentence-transformers (>=5.0.0) ; python_version < \"3.13\"", "tokenizers (>=0.19) ; python_version < \"3.13\"", "torch (>=2.6.0) ; python_version < \"3.13\"", "torchvision (>=0.17.0) ; python_version < \"3.13\"", "transformers (>=4.36.2) ; python_version < \"3.13\"", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] bedrock = ["boto3 (>=1.34.98,<2)", "botocore (>=1.34.110,<2)"] -cohere = ["cohere (>=5.9.4,<6.00)"] +cohere = ["cohere (>=5.9.4,<6.0)"] dev = ["dagger-io (>=0.1.1) ; python_version >= \"3.11\"", "ipykernel (>=6.25.0,<7)", "mypy (>=1.7.1,<2)", "pytest (>=8.2,<9.0)", "pytest-asyncio (>=0.24.0,<0.25)", "pytest-cov (>=4.1.0,<5)", "pytest-mock (>=3.12.0,<4)", "pytest-timeout", "pytest-xdist (>=3.5.0,<4)", "python-dotenv (>=1.0.0,<2)", "requests-mock (>=1.12.1,<2)", "ruff (>=0.11.2,<0.12)", "types-pyyaml (>=6.0.12.12,<7)", "types-requests (>=2.31.0,<3)"] docs = ["pydoc-markdown (>=4.8.2) ; python_version < \"3.12\""] fastembed = ["fastembed (>=0.3.0,<0.4) ; python_version < \"3.13\""] @@ -7222,6 +7230,7 @@ files = [ {file = "tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0"}, {file = "tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1"}, ] +markers = {main = "extra == \"extra-proxy\""} [[package]] name = "tornado" @@ -7994,4 +8003,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "1f3bbf967451633fb6290ba88980bdf4fbf83420024b14e862d1da717d903684" +content-hash = "22cdc8096e0c8296827734393f5ab6e66088f397b5295caa1d277466d1fde1e8" From 71b687e00a43486ae3171102772a3be02f72ba26 Mon Sep 17 00:00:00 2001 From: Alexey <5122340@mail.ru> Date: Wed, 18 Mar 2026 23:45:57 +0300 Subject: [PATCH 28/57] fix(proxy): sync normalized call_type into model_call_details for proxy-only errors --- litellm/proxy/utils.py | 13 ++-- tests/proxy_unit_tests/test_proxy_utils.py | 69 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 76662be175..e9946ebf97 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1863,28 +1863,33 @@ class ProxyLogging: ) input: Union[list, str, dict] = "" + normalized_call_type: Optional[str] = None if "messages" in request_data and isinstance( request_data["messages"], list ): input = request_data["messages"] litellm_logging_obj.model_call_details["messages"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.acompletion.value + normalized_call_type = CallTypes.acompletion.value elif "prompt" in request_data and isinstance(request_data["prompt"], str): input = request_data["prompt"] litellm_logging_obj.model_call_details["prompt"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.atext_completion.value + normalized_call_type = CallTypes.atext_completion.value elif "input" in request_data and isinstance(request_data["input"], list): input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input if litellm_logging_obj.call_type != CallTypes.pass_through.value: - litellm_logging_obj.call_type = CallTypes.aembedding.value + normalized_call_type = CallTypes.aembedding.value + if normalized_call_type is not None: + litellm_logging_obj.call_type = normalized_call_type + litellm_logging_obj.model_call_details["call_type"] = ( + normalized_call_type + ) # Pass-through endpoints are logged via the callback loop's # async_post_call_failure_hook — skip pre_call and failure handlers. if litellm_logging_obj.call_type == CallTypes.pass_through.value: return - litellm_logging_obj.pre_call( input=input, api_key="", diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index d77a7465c1..00d4cd24e4 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2278,6 +2278,75 @@ async def test_post_call_failure_hook_auth_error_llm_api_route(): mock_handle_logging.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_data, route, expected_call_type", + [ + ( + {"model": "bad-model", "messages": [{"role": "user", "content": "hello"}]}, + "/v1/chat/completions", + "acompletion", + ), + ( + {"model": "bad-model", "prompt": "hello"}, + "/v1/completions", + "atext_completion", + ), + ( + {"model": "bad-model", "input": ["hello"]}, + "/v1/embeddings", + "aembedding", + ), + ], +) +async def test_handle_logging_proxy_only_error_syncs_normalized_call_type( + request_data, route, expected_call_type +): + from fastapi import HTTPException + + from litellm.caching.caching import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import ProxyLogging + + cache = DualCache() + proxy_logging = ProxyLogging(user_api_key_cache=cache) + captured_logging_obj = {} + original_function_setup = litellm.utils.function_setup + + def _capture_function_setup(*args, **kwargs): + logging_obj, data = original_function_setup(*args, **kwargs) + captured_logging_obj["logging_obj"] = logging_obj + return logging_obj, data + + with patch( + "litellm.proxy.utils.litellm.utils.function_setup", + side_effect=_capture_function_setup, + ), patch.object( + Logging, "async_failure_handler", new=AsyncMock(return_value=None) + ), patch.object( + Logging, "failure_handler", return_value=None + ), patch( + "litellm.proxy.utils.threading.Thread" + ) as mock_thread: + mock_thread.return_value.start = Mock() + + await proxy_logging._handle_logging_proxy_only_error( + request_data=request_data, + user_api_key_dict=UserAPIKeyAuth( + api_key="test_key", + user_id="test_user", + token="test_token", + request_route=route, + ), + route=route, + original_exception=HTTPException(status_code=400, detail="bad request"), + ) + + logging_obj = captured_logging_obj["logging_obj"] + assert logging_obj.call_type == expected_call_type + assert logging_obj.model_call_details["call_type"] == expected_call_type + + @pytest.mark.asyncio async def test_during_call_hook_parallel_execution(): """ From 3ba18d708472869f7d6f7bbdb37c1cc2b471b56a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:38:58 -0700 Subject: [PATCH 29/57] [Refactor] UI - Playground: Extract ChatMessageBubble from ChatUI Extract the chat message bubble rendering (~165 lines) into a dedicated ChatMessageBubble component with 15 Vitest tests covering all display branches. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../chat_ui/ChatMessageBubble.test.tsx | 280 ++++++++++++++++++ .../playground/chat_ui/ChatMessageBubble.tsx | 214 +++++++++++++ .../components/playground/chat_ui/ChatUI.tsx | 171 +---------- 3 files changed, 503 insertions(+), 162 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx new file mode 100644 index 0000000000..368043abee --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx @@ -0,0 +1,280 @@ +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import ChatMessageBubble from "./ChatMessageBubble"; +import { EndpointType } from "./mode_endpoint_mapping"; +import { MessageType } from "./types"; + +// Mock child components to isolate bubble rendering logic +vi.mock("react-markdown", () => ({ + default: ({ children }: { children: string }) =>
{children}
, +})); + +vi.mock("react-syntax-highlighter", () => ({ + Prism: ({ children }: { children: string }) => 
{children}
, +})); + +vi.mock("react-syntax-highlighter/dist/esm/styles/prism", () => ({ + coy: {}, +})); + +vi.mock("./ReasoningContent", () => ({ + default: ({ reasoningContent }: { reasoningContent: string }) => ( +
{reasoningContent}
+ ), +})); + +vi.mock("./MCPEventsDisplay", () => ({ + default: ({ events }: { events: unknown[] }) => ( +
{events.length} events
+ ), +})); + +vi.mock("./SearchResultsDisplay", () => ({ + SearchResultsDisplay: ({ searchResults }: { searchResults: unknown[] }) => ( +
{searchResults.length} results
+ ), +})); + +vi.mock("./ResponseMetrics", () => ({ + default: ({ timeToFirstToken }: { timeToFirstToken?: number }) => ( +
TTFT: {timeToFirstToken}
+ ), +})); + +vi.mock("./A2AMetrics", () => ({ + default: ({ a2aMetadata }: { a2aMetadata: unknown }) => ( +
A2A
+ ), +})); + +vi.mock("./CodeInterpreterOutput", () => ({ + default: ({ code }: { code: string }) =>
{code}
, +})); + +vi.mock("./AudioRenderer", () => ({ + default: ({ message }: { message: MessageType }) => ( +
{typeof message.content === "string" ? message.content : ""}
+ ), +})); + +vi.mock("./ResponsesImageRenderer", () => ({ + default: () =>
, +})); + +vi.mock("./ChatImageRenderer", () => ({ + default: () =>
, +})); + +const defaultProps = { + isLastMessage: false, + endpointType: EndpointType.CHAT, + mcpEvents: [], + codeInterpreterResult: null, + accessToken: "test-token", +}; + +describe("ChatMessageBubble", () => { + it("should render a user message with right-aligned text", () => { + render( + , + ); + + expect(screen.getByText("user")).toBeInTheDocument(); + expect(screen.getByText("Hello")).toBeInTheDocument(); + }); + + it("should render an assistant message with left-aligned text", () => { + render( + , + ); + + expect(screen.getByText("assistant")).toBeInTheDocument(); + expect(screen.getByText("Hi there")).toBeInTheDocument(); + }); + + it("should show model badge for assistant messages when model is provided", () => { + render( + , + ); + + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + }); + + it("should not show model badge for user messages even when model is set", () => { + render( + , + ); + + expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); + }); + + it("should render markdown content via ReactMarkdown", () => { + render( + , + ); + + expect(screen.getByTestId("react-markdown")).toHaveTextContent("**bold text**"); + }); + + it("should render an image when isImage is true", () => { + render( + , + ); + + expect(screen.getByAltText("Generated image")).toHaveAttribute("src", "https://example.com/img.png"); + }); + + it("should render AudioRenderer when isAudio is true", () => { + render( + , + ); + + expect(screen.getByTestId("audio-renderer")).toBeInTheDocument(); + }); + + it("should show ReasoningContent when reasoningContent is present", () => { + render( + , + ); + + expect(screen.getByTestId("reasoning-content")).toHaveTextContent("thinking..."); + }); + + it("should show MCP events on the last assistant message for RESPONSES endpoint", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); + }); + + it("should not show MCP events when isLastMessage is false", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.queryByTestId("mcp-events-display")).not.toBeInTheDocument(); + }); + + it("should show SearchResultsDisplay when searchResults are present", () => { + render( + , + ); + + expect(screen.getByTestId("search-results-display")).toBeInTheDocument(); + }); + + it("should show ResponseMetrics when usage data is present and no a2aMetadata", () => { + render( + , + ); + + expect(screen.getByTestId("response-metrics")).toBeInTheDocument(); + }); + + it("should show A2AMetrics when a2aMetadata is present instead of ResponseMetrics", () => { + render( + , + ); + + expect(screen.getByTestId("a2a-metrics")).toBeInTheDocument(); + expect(screen.queryByTestId("response-metrics")).not.toBeInTheDocument(); + }); + + it("should show CodeInterpreterOutput on the last assistant message for RESPONSES endpoint", () => { + render( + , + ); + + expect(screen.getByTestId("code-interpreter-output")).toHaveTextContent("print('hello')"); + }); + + it("should render generated image from chat completions via message.image", () => { + render( + , + ); + + const images = screen.getAllByAltText("Generated image"); + expect(images.some((img) => img.getAttribute("src") === "https://example.com/generated.png")).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx new file mode 100644 index 0000000000..3e1913cb2b --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -0,0 +1,214 @@ +import { RobotOutlined, UserOutlined } from "@ant-design/icons"; +import React from "react"; +import ReactMarkdown from "react-markdown"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler"; +import A2AMetrics from "./A2AMetrics"; +import AudioRenderer from "./AudioRenderer"; +import ChatImageRenderer from "./ChatImageRenderer"; +import CodeInterpreterOutput from "./CodeInterpreterOutput"; +import { EndpointType } from "./mode_endpoint_mapping"; +import MCPEventsDisplay from "./MCPEventsDisplay"; +import type { MCPEvent } from "../../mcp_tools/types"; +import ReasoningContent from "./ReasoningContent"; +import ResponseMetrics from "./ResponseMetrics"; +import ResponsesImageRenderer from "./ResponsesImageRenderer"; +import { SearchResultsDisplay } from "./SearchResultsDisplay"; +import { MessageType } from "./types"; + +interface ChatMessageBubbleProps { + message: MessageType; + /** Whether this is the last message in the chat history. */ + isLastMessage: boolean; + endpointType: string; + /** MCP events to display on the last assistant message. */ + mcpEvents: MCPEvent[]; + /** Code interpreter result to display on the last assistant message. */ + codeInterpreterResult: CodeInterpreterResult | null; + /** API key used to fetch code interpreter file downloads. */ + accessToken: string; +} + +function ChatMessageBubble({ + message, + isLastMessage, + endpointType, + mcpEvents, + codeInterpreterResult, + accessToken, +}: ChatMessageBubbleProps) { + const isUser = message.role === "user"; + + return ( +
+
+ {/* Header: role icon + name + model badge */} +
+
+ {isUser ? ( + + ) : ( + + )} +
+ {message.role} + {message.role === "assistant" && message.model && ( + + {message.model} + + )} +
+ + {/* Reasoning content (chain-of-thought) */} + {message.reasoningContent && } + + {/* MCP events at the start of the last assistant message */} + {message.role === "assistant" && + isLastMessage && + mcpEvents.length > 0 && + (endpointType === EndpointType.RESPONSES || endpointType === EndpointType.CHAT) && ( +
+ +
+ )} + + {/* Search results */} + {message.role === "assistant" && message.searchResults && ( + + )} + + {/* Code Interpreter output for the last assistant message */} + {message.role === "assistant" && + isLastMessage && + codeInterpreterResult && + endpointType === EndpointType.RESPONSES && ( + + )} + + {/* Message body */} +
+ {message.isImage ? ( + Generated image + ) : message.isAudio ? ( + + ) : ( + <> + {/* Attached image for user messages based on endpoint */} + {endpointType === EndpointType.RESPONSES && } + {endpointType === EndpointType.CHAT && } + + & { + inline?: boolean; + node?: unknown; + }) { + const match = /language-(\w+)/.exec(className || ""); + return !inline && match ? ( + } + language={match[1]} + PreTag="div" + className="rounded-md my-2" + wrapLines={true} + wrapLongLines={true} + {...props} + > + {String(children).replace(/\n$/, "")} + + ) : ( + + {children} + + ); + }, + pre: ({ node, ...props }) => ( + 
+                  ),
+                }}
+              >
+                {typeof message.content === "string" ? message.content : ""}
+              
+
+              {/* Generated image from chat completions */}
+              {message.image && (
+                

+                  Generated image
+                

+              )}
+            
+          )}
+
+          {/* Response metrics */}
+          {message.role === "assistant" &&
+            (message.timeToFirstToken || message.totalLatency || message.usage) &&
+            !message.a2aMetadata && (
+              
+            )}
+
+          {/* A2A Metrics */}
+          {message.role === "assistant" && message.a2aMetadata && (
+            
+          )}
+
+
+
+ ); +} + +export default ChatMessageBubble; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index bc0d52cf58..ec37a9e1e9 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -63,6 +63,7 @@ import EndpointSelector from "./EndpointSelector"; import FilePreviewCard from "./FilePreviewCard"; import MCPEventsDisplay from "./MCPEventsDisplay"; import type { MCPEvent } from "../../mcp_tools/types"; +import ChatMessageBubble from "./ChatMessageBubble"; import { EndpointType, getEndpointType } from "./mode_endpoint_mapping"; import ReasoningContent from "./ReasoningContent"; import ResponseMetrics, { TokenUsage } from "./ResponseMetrics"; @@ -1932,168 +1933,14 @@ const ChatUI: React.FC = ({ {chatHistory.map((message, index) => (
-
-
-
-
- {message.role === "user" ? ( - - ) : ( - - )} -
- {message.role} - {message.role === "assistant" && message.model && ( - - {message.model} - - )} -
- {message.reasoningContent && } - - {/* Show MCP events at the start of assistant messages */} - {message.role === "assistant" && - index === chatHistory.length - 1 && - mcpEvents.length > 0 && - (endpointType === EndpointType.RESPONSES || endpointType === EndpointType.CHAT) && ( -
- -
- )} - - {/* Show search results at the start of assistant messages */} - {message.role === "assistant" && message.searchResults && ( - - )} - - {/* Show Code Interpreter output for the last assistant message */} - {message.role === "assistant" && - index === chatHistory.length - 1 && - codeInterpreter.result && - endpointType === EndpointType.RESPONSES && ( - - )} - -
- {message.isImage ? ( - Generated image - ) : message.isAudio ? ( - - ) : ( - <> - {/* Show attached image for user messages based on current endpoint */} - {endpointType === EndpointType.RESPONSES && } - {endpointType === EndpointType.CHAT && } - - & { - inline?: boolean; - node?: any; - }) { - const match = /language-(\w+)/.exec(className || ""); - return !inline && match ? ( - - {String(children).replace(/\n$/, "")} - - ) : ( - - {children} - - ); - }, - pre: ({ node, ...props }) => ( - 
-                                ),
-                              }}
-                            >
-                              {typeof message.content === "string" ? message.content : ""}
-                            
-
-                            {/* Show generated image from chat completions */}
-                            {message.image && (
-                              

-                                Generated image
-                              

-                            )}
-                          
-                        )}
-
-                        {message.role === "assistant" &&
-                          (message.timeToFirstToken || message.totalLatency || message.usage) &&
-                          !message.a2aMetadata && (
-                            
-                          )}
-
-                        {/* A2A Metrics - show for A2A agent responses */}
-                        {message.role === "assistant" && message.a2aMetadata && (
-                          
-                        )}
-
-
-
+
))} From b55cb249fe276996e9ca351e31d08e7cff4a9542 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:46:41 -0700 Subject: [PATCH 30/57] Address Greptile feedback: use EndpointType enum, add CHAT MCP test - Narrow endpointType prop from string to EndpointType enum - Add missing test for MCP events on CHAT endpoint Co-Authored-By: Claude Opus 4.6 (1M context) --- .../chat_ui/ChatMessageBubble.test.tsx | 16 ++++++++++++++++ .../playground/chat_ui/ChatMessageBubble.tsx | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx index 368043abee..70c3fdd4f2 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.test.tsx @@ -180,6 +180,22 @@ describe("ChatMessageBubble", () => { expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); }); + it("should show MCP events on the last assistant message for CHAT endpoint", () => { + const mcpEvents = [{ type: "tool_call", item_id: "1" }]; + + render( + , + ); + + expect(screen.getByTestId("mcp-events-display")).toHaveTextContent("1 events"); + }); + it("should not show MCP events when isLastMessage is false", () => { const mcpEvents = [{ type: "tool_call", item_id: "1" }]; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx index 3e1913cb2b..148cd082e2 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -21,7 +21,7 @@ interface ChatMessageBubbleProps { message: MessageType; /** Whether this is the last message in the chat history. */ isLastMessage: boolean; - endpointType: string; + endpointType: EndpointType; /** MCP events to display on the last assistant message. */ mcpEvents: MCPEvent[]; /** Code interpreter result to display on the last assistant message. */ From f6cd0a827ae84cffa838eac859eaea504bd6464a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 16:53:29 -0700 Subject: [PATCH 31/57] fix: /key/update returns 404 (not 401) for nonexistent body key The /key/update endpoint's get_data() call raises a 401 when the body `key` field doesn't exist in the DB, because get_data() treats the token as an auth credential. This caused the auth layer to resolve the body key instead of the Authorization header bearer token. Replace prisma_client.get_data() with direct Prisma find_unique() in both _get_and_validate_existing_key() and update_key_fn(), matching the pattern used in the /key/block and /key/unblock fix (PR #23977). Also fix the incorrect "Team not found" error message in update_key_fn. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../key_management_endpoints.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index ec33816b78..2884c8d374 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1662,16 +1662,23 @@ async def _get_and_validate_existing_key( detail={"error": "Database not connected"}, ) - existing_key_row = await prisma_client.get_data( - token=token, - table_name="key", - query_type="find_unique", + from litellm.proxy.proxy_server import hash_token + + if token.startswith("sk-"): + hashed_token = hash_token(token=token) + else: + hashed_token = token + + existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} ) if existing_key_row is None: - raise HTTPException( - status_code=404, - detail={"error": f"Key not found: {token}"}, + raise ProxyException( + message=f"Key not found. Passed key={token}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) return existing_key_row @@ -2112,14 +2119,23 @@ async def update_key_fn( if prisma_client is None: raise Exception("Not connected to DB!") - existing_key_row = await prisma_client.get_data( - token=data.key, table_name="key", query_type="find_unique" + from litellm.proxy.proxy_server import hash_token + + if data.key.startswith("sk-"): + hashed_token = hash_token(token=data.key) + else: + hashed_token = data.key + + existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( + where={"token": hashed_token} ) if existing_key_row is None: - raise HTTPException( - status_code=404, - detail={"error": f"Team not found, passed team_id={data.team_id}"}, + raise ProxyException( + message=f"Key not found. Passed key={data.key}", + type=ProxyErrorTypes.not_found_error, + param="key", + code=status.HTTP_404_NOT_FOUND, ) await _validate_update_key_data( From ebe329cdce15edfd3f0c783ae2bad9053a66bd83 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:02:18 -0700 Subject: [PATCH 32/57] Fix build: use `as any` for SyntaxHighlighter style prop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches the cast used in ChatUI.tsx — the react-syntax-highlighter type definitions don't accept CSSProperties directly. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/playground/chat_ui/ChatMessageBubble.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx index 148cd082e2..15978c17f7 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatMessageBubble.tsx @@ -143,7 +143,7 @@ function ChatMessageBubble({ const match = /language-(\w+)/.exec(className || ""); return !inline && match ? ( } + style={coy as any} language={match[1]} PreTag="div" className="rounded-md my-2" From eceb4981b851659e6ca2152660216577e5f06feb Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:03:43 -0700 Subject: [PATCH 33/57] fix: address review feedback - dedup logic, use module-level helper, add test - Deduplicate: update_key_fn now delegates to _get_and_validate_existing_key() instead of inlining its own copy of the lookup logic - Use _hash_token_if_needed (already imported at module level) instead of inline `from proxy_server import hash_token` + manual conditional - Fix stale docstring: _get_and_validate_existing_key raises ProxyException, not HTTPException - Add unit test: test_update_key_nonexistent_key_returns_404 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../key_management_endpoints.py | 32 ++---------- .../test_key_management_endpoints.py | 50 +++++++++++++++++++ 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2884c8d374..7be1a5a5e0 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1654,7 +1654,7 @@ async def _get_and_validate_existing_key( LiteLLM_VerificationToken: The existing key row Raises: - HTTPException: If key is not found + ProxyException: 404 if key is not found """ if prisma_client is None: raise HTTPException( @@ -1662,12 +1662,7 @@ async def _get_and_validate_existing_key( detail={"error": "Database not connected"}, ) - from litellm.proxy.proxy_server import hash_token - - if token.startswith("sk-"): - hashed_token = hash_token(token=token) - else: - hashed_token = token + hashed_token = _hash_token_if_needed(token=token) existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( where={"token": hashed_token} @@ -2116,28 +2111,11 @@ async def update_key_fn( key = data_json.pop("key") # get the row from db - if prisma_client is None: - raise Exception("Not connected to DB!") - - from litellm.proxy.proxy_server import hash_token - - if data.key.startswith("sk-"): - hashed_token = hash_token(token=data.key) - else: - hashed_token = data.key - - existing_key_row = await prisma_client.db.litellm_verificationtoken.find_unique( - where={"token": hashed_token} + existing_key_row = await _get_and_validate_existing_key( + token=data.key, + prisma_client=prisma_client, ) - if existing_key_row is None: - raise ProxyException( - message=f"Key not found. Passed key={data.key}", - type=ProxyErrorTypes.not_found_error, - param="key", - code=status.HTTP_404_NOT_FOUND, - ) - await _validate_update_key_data( data=data, existing_key_row=existing_key_row, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 3bff35fbd5..48a1f7936c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -1678,6 +1678,56 @@ async def test_unblock_key_nonexistent_key_returns_404(monkeypatch): mock_prisma_client.db.litellm_verificationtoken.update.assert_not_called() +@pytest.mark.asyncio +async def test_update_key_nonexistent_key_returns_404(monkeypatch): + """ + Test that update_key_fn returns 404 (not misleading 401) when the body + key doesn't exist in the database, even when the caller is authenticated + as a proxy admin via the Authorization header. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + update_key_fn, + ) + + mock_prisma_client = AsyncMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + # find_unique returns None → key does not exist + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr( + "litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache + ) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + + mock_request = MagicMock() + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin_user" + ) + + data = UpdateKeyRequest(key="sk-does-not-exist-key") + + with pytest.raises(ProxyException) as exc_info: + await update_key_fn( + request=mock_request, + data=data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + assert exc_info.value.code == "404" + assert "not found" in str(exc_info.value.message).lower() + assert "Authentication Error" not in str(exc_info.value.message) + + @pytest.mark.asyncio async def test_block_key_existing_key_succeeds(monkeypatch): """ From 0b63979d4572bfa6225df9bc62f9fbae8d6b3710 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 18 Mar 2026 17:04:42 -0700 Subject: [PATCH 34/57] Fix build: cast endpointType to EndpointType at call site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChatUI stores endpointType as string but the narrowed prop expects EndpointType — add explicit cast at the call site. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/components/playground/chat_ui/ChatUI.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index ec37a9e1e9..ef57a75062 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -1936,7 +1936,7 @@ const ChatUI: React.FC = ({ Date: Wed, 18 Mar 2026 17:56:25 -0700 Subject: [PATCH 35/57] [Feature] UI - Leftnav: Add external link icon to Learning Resources Add ExportOutlined icon next to nav items that link to external pages, making it clear to users when a link opens in a new tab. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/components/leftnav.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index d01fc06bc0..d3789fcffa 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -13,6 +13,7 @@ import { CreditCardOutlined, DatabaseOutlined, ExperimentOutlined, + ExportOutlined, FileTextOutlined, FolderOutlined, KeyOutlined, @@ -400,7 +401,7 @@ const Sidebar: React.FC = ({ setPage, defaultSelectedKey, collapse onClick={(e) => e.stopPropagation()} style={{ color: "inherit", textDecoration: "none" }} > - {label} + {label} ); } From 4770b657e15a25815eccf211f87843db356625e8 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 18 Mar 2026 22:05:27 -0300 Subject: [PATCH 36/57] =?UTF-8?q?refactor:=20extract=20duplicated=20stdout?= =?UTF-8?q?/stderr=20=E2=86=92=20logs=20logic=20to=20shared=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm/llms/anthropic/chat/handler.py | 15 ++------------- litellm/llms/anthropic/chat/transformation.py | 17 ++--------------- litellm/types/responses/main.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 70ecf91725..7dce72f1e8 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -50,7 +50,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import ( OutputCodeInterpreterCall, - OutputCodeInterpreterCallLog, + build_code_interpreter_log_outputs, ) from litellm.types.utils import ( Delta, @@ -708,20 +708,9 @@ class ModelResponseIterator: continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) - if isinstance(content, dict): - parts = [] - if content.get("stdout"): - parts.append(content["stdout"]) - if content.get("stderr"): - parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) - else: - logs = "" + log_outputs = build_code_interpreter_log_outputs(content) tool_input = self._server_tool_inputs.get(call_id, {}) code = tool_input.get("command", "") if isinstance(tool_input, dict) else "" - log_outputs = ( - [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None - ) results.append( OutputCodeInterpreterCall( type="code_interpreter_call", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index ca101df0e9..bbc73fcfd4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -61,7 +61,7 @@ from litellm.types.utils import ( ) from litellm.types.responses.main import ( OutputCodeInterpreterCall, - OutputCodeInterpreterCallLog, + build_code_interpreter_log_outputs, ) from litellm.utils import ( ModelResponse, @@ -1771,20 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): continue call_id = tr.get("tool_use_id", "") content = tr.get("content", {}) - if isinstance(content, dict): - parts = [] - if content.get("stdout"): - parts.append(content["stdout"]) - if content.get("stderr"): - parts.append(f"STDERR: {content['stderr']}") - logs = "".join(parts) - else: - logs = "" - log_outputs = ( - [OutputCodeInterpreterCallLog(type="logs", logs=logs)] - if logs - else None - ) + log_outputs = build_code_interpreter_log_outputs(content) code_interpreter_results.append( OutputCodeInterpreterCall( type="code_interpreter_call", diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index e46857565c..ebd2ad5b5a 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -67,6 +67,24 @@ class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): outputs: Optional[List[OutputCodeInterpreterCallLog]] +def build_code_interpreter_log_outputs( + content: Any, +) -> Optional[List[OutputCodeInterpreterCallLog]]: + """Convert Anthropic bash_code_execution stdout/stderr to log outputs. + + Shared by streaming (handler.py) and non-streaming (transformation.py) paths. + """ + if not isinstance(content, dict): + return None + parts = [] + if content.get("stdout"): + parts.append(content["stdout"]) + if content.get("stderr"): + parts.append(f"STDERR: {content['stderr']}") + logs = "".join(parts) + return [OutputCodeInterpreterCallLog(type="logs", logs=logs)] if logs else None + + class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): """ Generic response API output item From 8969a3d1763e7e2bb3b0c48dbb855780cf992ebe Mon Sep 17 00:00:00 2001 From: xianren Date: Thu, 19 Mar 2026 09:10:21 +0800 Subject: [PATCH 37/57] Fixed thinking blocks dropped when thinking field is null (#24026) The check `content.get("thinking", None) is not None` incorrectly drops thinking blocks when the `thinking` key is explicitly null or absent. Changed to `content.get("type") == "thinking"` to match the fix already applied in the experimental pass-through path (PR #15501). Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/llms/anthropic/chat/transformation.py | 2 +- .../test_anthropic_chat_transformation.py | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 47cdd8287e..7552becef6 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1522,7 +1522,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): tool_results = [] tool_results.append(content) - elif content.get("thinking", None) is not None: + elif content.get("type") == "thinking": if thinking_blocks is None: thinking_blocks = [] thinking_blocks.append(cast(ChatCompletionThinkingBlock, content)) diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index a95b9413b9..7f5b7d6158 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3321,3 +3321,54 @@ def test_map_tool_helper_empty_parameters_get_default(): assert result is not None assert result["input_schema"]["type"] == "object" assert result["input_schema"].get("properties") == {} + + +def test_extract_response_content_thinking_block_null_thinking(): + """ + Test that thinking blocks are not dropped when the 'thinking' field is null + or missing. Regression test for https://github.com/BerriAI/litellm/issues/24026 + """ + config = AnthropicConfig() + + # Case 1: thinking key is explicitly null + completion_response_null = { + "content": [ + {"type": "thinking", "thinking": None, "signature": "sig123"}, + {"type": "text", "text": "Hello"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_null + ) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking=null" + assert len(thinking_blocks) == 1 + assert "Hello" in text + + # Case 2: thinking key is absent entirely + completion_response_missing = { + "content": [ + {"type": "thinking", "signature": "sig456"}, + {"type": "text", "text": "World"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_missing + ) + assert thinking_blocks is not None, "thinking blocks should not be None when thinking key is absent" + assert len(thinking_blocks) == 1 + assert "World" in text + + # Case 3: thinking key has actual content (should still work) + completion_response_text = { + "content": [ + {"type": "thinking", "thinking": "Let me think...", "signature": "sig789"}, + {"type": "text", "text": "Done"}, + ] + } + text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content( + completion_response_text + ) + assert thinking_blocks is not None + assert len(thinking_blocks) == 1 + assert thinking_blocks[0]["thinking"] == "Let me think..." + assert "Done" in text From bd0c3bfdc4d9cd364c8b68b975ed8398c55b0dc0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 18 Mar 2026 20:58:41 -0700 Subject: [PATCH 38/57] fix: fix logging for response incomplete streaming --- litellm/litellm_core_utils/litellm_logging.py | 294 ++++++++---------- .../anthropic_passthrough_logging_handler.py | 40 ++- litellm/responses/streaming_iterator.py | 52 ++-- .../test_litellm_logging.py | 192 ++++++++++-- 4 files changed, 342 insertions(+), 236 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4e63dd7076..5e34a4c992 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,47 +12,28 @@ import time import traceback from datetime import datetime as dt_object from functools import lru_cache -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Literal, - Optional, - Tuple, - Type, - Union, - cast, -) +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, + Optional, Tuple, Type, Union, cast) from httpx import Response from pydantic import BaseModel import litellm -from litellm import ( - _custom_logger_compatible_callbacks_literal, - json_logs, - log_raw_request_response, - turn_off_message_logging, -) +from litellm import (_custom_logger_compatible_callbacks_literal, json_logs, + log_raw_request_response, turn_off_message_logging) from litellm._logging import _is_debugging_on, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler -from litellm.constants import ( - DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - SENTRY_DENYLIST, - SENTRY_PII_DENYLIST, -) -from litellm.cost_calculator import ( - RealtimeAPITokenUsageProcessor, - _select_model_name_for_cost_calc, -) +from litellm.constants import (DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + SENTRY_DENYLIST, SENTRY_PII_DENYLIST) +from litellm.cost_calculator import (RealtimeAPITokenUsageProcessor, + _select_model_name_for_cost_calc) from litellm.integrations.agentops import AgentOps -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import \ + AnthropicCacheControlHook from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger @@ -61,70 +42,48 @@ from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params -from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( - StandardBuiltInToolCostTracking, -) -from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import \ + StandardBuiltInToolCostTracking +from litellm.litellm_core_utils.logging_utils import \ + truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, - redact_message_input_output_from_logging, -) + redact_message_input_output_from_logging) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.llms.openai import ( - AllMessageValues, - Batch, - FineTuningJob, - HttpxBinaryResponseContent, - OpenAIFileObject, - OpenAIModerationResponse, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIResponse, -) +from litellm.types.llms.openai import (AllMessageValues, Batch, FineTuningJob, + HttpxBinaryResponseContent, + OpenAIFileObject, + OpenAIModerationResponse, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( - CachingDetails, - CallTypes, - CostBreakdown, - CostResponseTypes, - CustomPricingLiteLLMParams, - DynamicPromptManagementParamLiteral, - EmbeddingResponse, - GuardrailStatus, - ImageResponse, - LiteLLMBatch, - LiteLLMLoggingBaseClass, - LiteLLMRealtimeStreamLoggingObject, - ModelResponse, - ModelResponseStream, - RawRequestTypedDict, - StandardBuiltInToolsParams, - StandardCallbackDynamicParams, - StandardLoggingAdditionalHeaders, - StandardLoggingHiddenParams, - StandardLoggingMCPToolCall, - StandardLoggingMetadata, - StandardLoggingModelCostFailureDebugInformation, - StandardLoggingModelInformation, - StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, - StandardLoggingPayloadStatus, + CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, + CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, + EmbeddingResponse, GuardrailStatus, ImageResponse, LiteLLMBatch, + LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, ModelResponse, + ModelResponseStream, RawRequestTypedDict, StandardBuiltInToolsParams, + StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, + StandardLoggingHiddenParams, StandardLoggingMCPToolCall, + StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, + StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, StandardLoggingPayloadStatus, StandardLoggingPayloadStatusFields, - StandardLoggingPromptManagementMetadata, - StandardLoggingVectorStoreRequest, - TextCompletionResponse, - TranscriptionResponse, - Usage, -) + StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, + TextCompletionResponse, TranscriptionResponse, Usage) from litellm.types.videos.main import VideoObject -from litellm.utils import _get_base_model_from_metadata, executor, print_verbose +from litellm.utils import (_get_base_model_from_metadata, executor, + print_verbose) from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -146,7 +105,8 @@ from ..integrations.humanloop import HumanloopLogger from ..integrations.lago import LagoLogger from ..integrations.langfuse.langfuse import LangFuseLogger from ..integrations.langfuse.langfuse_handler import LangFuseHandler -from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement +from ..integrations.langfuse.langfuse_prompt_management import \ + LangfusePromptManagement from ..integrations.langsmith import LangsmithLogger from ..integrations.litellm_agent import LiteLLMAgentModelResolver from ..integrations.literal_ai import LiteralAILogger @@ -161,34 +121,30 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers -from .initialize_dynamic_callback_params import ( - initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, -) +from .initialize_dynamic_callback_params import \ + initialize_standard_callback_dynamic_params as \ + _initialize_standard_callback_dynamic_params from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import \ + BasePassthroughConfig try: - from litellm_enterprise.enterprise_callbacks.callback_controls import ( - EnterpriseCallbackControls, - ) - from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( - PagerDutyAlerting, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( - ResendEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( - SendGridEmailLogger, - ) - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( - SMTPEmailLogger, - ) - from litellm_enterprise.litellm_core_utils.litellm_logging import ( - StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup, - ) + from litellm_enterprise.enterprise_callbacks.callback_controls import \ + EnterpriseCallbackControls + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import \ + PagerDutyAlerting + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ + ResendEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ + SendGridEmailLogger + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ + SMTPEmailLogger + from litellm_enterprise.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup - from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger + from litellm.integrations.generic_api.generic_api_callback import \ + GenericAPILogger EnterpriseStandardLoggingPayloadSetupVAR: Optional[ Type[EnterpriseStandardLoggingPayloadSetup] @@ -516,6 +472,23 @@ class Logging(LiteLLMLoggingBaseClass): ), ) + def get_router_model_id(self) -> Optional[str]: + """Extract the router deployment model_id from litellm_params. + + Checks both litellm_metadata and metadata for model_info.id. + Used by cost calculators to look up custom pricing registered + under the deployment's model_info.id in litellm.model_cost. + """ + if not hasattr(self, "litellm_params"): + return None + for key in ("litellm_metadata", "metadata"): + meta = self.litellm_params.get(key, {}) or {} + info = meta.get("model_info", {}) or {} + model_id = info.get("id") + if model_id is not None: + return model_id + return None + def update_environment_variables( self, litellm_params: Dict, @@ -1458,16 +1431,8 @@ class Logging(LiteLLMLoggingBaseClass): # Fallback: extract router_model_id from litellm_params when not available # from the result object. ResponsesAPIResponse objects (used by /v1/responses # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. - if router_model_id is None and hasattr(self, "litellm_params"): - for metadata_key in ("litellm_metadata", "metadata"): - _metadata: dict = ( - self.litellm_params.get(metadata_key, {}) or {} - ) - _model_info: dict = _metadata.get("model_info", {}) or {} - _model_id = _model_info.get("id") - if _model_id is not None: - router_model_id = _model_id - break + if router_model_id is None: + router_model_id = self.get_router_model_id() ## RESPONSE COST ## custom_pricing = use_custom_pricing_for_model( @@ -1758,9 +1723,8 @@ class Logging(LiteLLMLoggingBaseClass): ) standard_logging_payload["response"] = response_dict elif isinstance(result, TranscriptionResponse): - from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( - TranscriptionUsageObjectTransformation, - ) + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import \ + TranscriptionUsageObjectTransformation result = result.model_copy() transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore @@ -2443,9 +2407,8 @@ class Logging(LiteLLMLoggingBaseClass): ): # polling job will query these frequently, don't spam db logs return - from litellm.proxy.openai_files_endpoints.common_utils import ( - _is_base64_encoded_unified_file_id, - ) + from litellm.proxy.openai_files_endpoints.common_utils import \ + _is_base64_encoded_unified_file_id # check if file id is a unified file id is_base64_unified_file_id = _is_base64_encoded_unified_file_id(result.id) @@ -3321,7 +3284,7 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, TextCompletionResponse): return result - elif isinstance(result, ResponseCompletedEvent): + elif isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)): ## return unified Usage object if isinstance(result.response.usage, ResponseAPIUsage): transformed_usage = ( @@ -3588,7 +3551,8 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "s3": s3Logger = S3Logger() elif callback == "wandb": - from litellm.integrations.weights_biases import WeightsBiasesLogger + from litellm.integrations.weights_biases import \ + WeightsBiasesLogger weightsBiasesLogger = WeightsBiasesLogger() elif callback == "logfire": @@ -3652,7 +3616,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_posthog_logger) return _posthog_logger # type: ignore elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import BraintrustLogger + from litellm.integrations.braintrust_logging import \ + BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -3773,9 +3738,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _opik_logger # type: ignore elif logging_integration == "arize": from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) arize_config = ArizeLogger.get_arize_config() if arize_config.endpoint is None: @@ -3802,9 +3765,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) arize_phoenix_config = ArizePhoenixLogger.get_arize_phoenix_config() otel_config = OpenTelemetryConfig( @@ -3858,9 +3819,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "levo": from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) levo_config = LevoLogger.get_levo_config() otel_config = OpenTelemetryConfig( @@ -3909,7 +3868,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(galileo_logger) return galileo_logger # type: ignore elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import \ + CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -3929,7 +3889,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(focus_logger) return focus_logger # type: ignore elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import VantageLogger + from litellm.integrations.vantage.vantage_logger import \ + VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -3949,9 +3910,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) logfire_base_url = os.getenv( "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev" @@ -3969,9 +3928,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import ( - _PROXY_DynamicRateLimitHandler, - ) + from litellm.proxy.hooks.dynamic_rate_limiter import \ + _PROXY_DynamicRateLimitHandler for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): @@ -3993,9 +3951,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( - _PROXY_DynamicRateLimitHandlerV3, - ) + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ + _PROXY_DynamicRateLimitHandlerV3 for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4021,9 +3978,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 raise ValueError("LANGTRACE_API_KEY not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, - OpenTelemetryConfig, - ) + OpenTelemetry, OpenTelemetryConfig) otel_config = OpenTelemetryConfig( exporter="otlp_http", @@ -4059,7 +4014,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": - from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger + from litellm.integrations.langfuse.langfuse_otel import \ + LangfuseOtelLogger for callback in _in_memory_loggers: if ( @@ -4077,9 +4033,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "weave_otel": from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( - WeaveOtelLogger, - get_weave_otel_config, - ) + WeaveOtelLogger, get_weave_otel_config) weave_otel_config = get_weave_otel_config() @@ -4115,9 +4069,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(anthropic_cache_control_hook) return anthropic_cache_control_hook # type: ignore elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( - VectorStorePreCallHook, - ) + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ + VectorStorePreCallHook for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -4177,9 +4130,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dotprompt_logger) return dotprompt_logger # type: ignore elif logging_integration == "bitbucket": - from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( - BitBucketPromptManager, - ) + from litellm.integrations.bitbucket.bitbucket_prompt_manager import \ + BitBucketPromptManager for callback in _in_memory_loggers: if isinstance(callback, BitBucketPromptManager): @@ -4196,9 +4148,8 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(bitbucket_logger) return bitbucket_logger # type: ignore elif logging_integration == "gitlab": - from litellm.integrations.gitlab.gitlab_prompt_manager import ( - GitLabPromptManager, - ) + from litellm.integrations.gitlab.gitlab_prompt_manager import \ + GitLabPromptManager for callback in _in_memory_loggers: if isinstance(callback, GitLabPromptManager): @@ -4286,7 +4237,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, OpenMeterLogger): return callback elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import BraintrustLogger + from litellm.integrations.braintrust_logging import \ + BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -4296,7 +4248,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, GalileoObserve): return callback elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import \ + CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -4310,7 +4263,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 ): # exact match; exclude subclasses like VantageLogger return callback elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import VantageLogger + from litellm.integrations.vantage.vantage_logger import \ + VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -4410,17 +4364,15 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import ( - _PROXY_DynamicRateLimitHandler, - ) + from litellm.proxy.hooks.dynamic_rate_limiter import \ + _PROXY_DynamicRateLimitHandler for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( - _PROXY_DynamicRateLimitHandlerV3, - ) + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ + _PROXY_DynamicRateLimitHandlerV3 for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4452,9 +4404,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, AnthropicCacheControlHook): return callback elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( - VectorStorePreCallHook, - ) + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ + VectorStorePreCallHook for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -5626,7 +5577,6 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal - # used for unit testing from typing import Any, Dict, List, Optional, Union diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 20d06b7d53..3241c1ca93 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -6,20 +6,23 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.chat.handler import ( - ModelResponseIterator as AnthropicModelResponseIterator, -) +from litellm.llms.anthropic.chat.handler import \ + ModelResponseIterator as AnthropicModelResponseIterator from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.types.passthrough_endpoints.pass_through_endpoints import ( - PassthroughStandardLoggingPayload, -) -from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse +from litellm.types.passthrough_endpoints.pass_through_endpoints import \ + PassthroughStandardLoggingPayload +from litellm.types.utils import (LiteLLMBatch, ModelResponse, + TextCompletionResponse) if TYPE_CHECKING: - from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType + from litellm.types.passthrough_endpoints.pass_through_endpoints import \ + EndpointType from ..success_handler import PassThroughEndpointLogging else: @@ -124,10 +127,21 @@ class AnthropicPassthroughLoggingHandler: if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): model_for_cost = f"{custom_llm_provider}/{model}" + router_model_id = logging_obj.get_router_model_id() + custom_pricing = use_custom_pricing_for_model( + litellm_params=( + logging_obj.litellm_params + if hasattr(logging_obj, "litellm_params") + else None + ) + ) + response_cost = litellm.completion_cost( completion_response=litellm_model_response, model=model_for_cost, custom_llm_provider=custom_llm_provider, + custom_pricing=custom_pricing, + router_model_id=router_model_id, ) kwargs["response_cost"] = response_cost @@ -319,9 +333,8 @@ class AnthropicPassthroughLoggingHandler: import base64 from litellm._uuid import uuid - from litellm.llms.anthropic.batches.transformation import ( - AnthropicBatchesConfig, - ) + from litellm.llms.anthropic.batches.transformation import \ + AnthropicBatchesConfig from litellm.types.utils import Choices, SpecialEnums try: @@ -537,7 +550,8 @@ class AnthropicPassthroughLoggingHandler: managed_files_hook, "store_unified_object_id" ): # Create a mock user API key dict for the managed object storage - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy._types import (LitellmUserRoles, + UserAPIKeyAuth) user_api_key_dict = UserAPIKeyAuth( user_id=kwargs.get("user_id", "default-user"), diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 073ee92606..a6a1074067 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,31 +8,29 @@ from typing import Any, Dict, List, Optional import httpx import litellm -from litellm.constants import ( - LITELLM_MAX_STREAMING_DURATION_SECONDS, - STREAM_SSE_DONE_STRING, -) +from litellm.constants import (LITELLM_MAX_STREAMING_DURATION_SECONDS, + STREAM_SSE_DONE_STRING) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, -) +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import \ + get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import \ + update_response_metadata from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import \ + BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import ( - OutputTextDeltaEvent, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, - ResponsesAPIStreamingResponse, -) +from litellm.types.llms.openai import (OutputTextDeltaEvent, ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse) from litellm.types.utils import CallTypes -from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook +from litellm.utils import (CustomStreamWrapper, + async_post_call_success_deployment_hook) class BaseResponsesAPIStreamingIterator: @@ -166,11 +164,16 @@ class BaseResponsesAPIStreamingIterator: ) setattr(item, "encrypted_content", wrapped_content) - # Store the completed response + # Store the completed response (also for incomplete/failed so logging still fires) + _chunk_type = getattr(openai_responses_api_chunk, "type", None) if ( openai_responses_api_chunk - and getattr(openai_responses_api_chunk, "type", None) - == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + and _chunk_type + in ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ResponsesAPIStreamEvents.RESPONSE_FAILED, + ) ): self.completed_response = openai_responses_api_chunk # Add cost to usage object if include_cost_in_streaming_usage is True @@ -694,7 +697,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): # --------------------------------------------------------------------------- from litellm._logging import verbose_logger -from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor +from litellm.litellm_core_utils.thread_pool_executor import \ + executor as _ws_executor RESPONSES_WS_LOGGED_EVENT_TYPES = [ "response.created", diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index fe4851283f..0f950f6da7 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -11,7 +11,8 @@ sys.path.insert( import time from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST -from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.litellm_core_utils.litellm_logging import \ + Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import set_callbacks from litellm.types.utils import ModelResponse, TextCompletionResponse @@ -139,7 +140,8 @@ def test_sentry_environment(): def test_use_custom_pricing_for_model(): - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "custom_llm_provider": "azure", @@ -154,7 +156,8 @@ def test_use_custom_pricing_for_model_via_litellm_metadata(): Generic API call routes (/messages, /responses) store model_info under litellm_metadata, not metadata. Regression test for #23185. """ - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "litellm_metadata": { @@ -170,7 +173,8 @@ def test_use_custom_pricing_for_model_via_litellm_metadata(): def test_use_custom_pricing_not_detected_litellm_metadata_no_pricing(): """Should return False when litellm_metadata.model_info has no pricing keys.""" - from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model litellm_params = { "litellm_metadata": { @@ -186,7 +190,8 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): does not carry _hidden_params (e.g. ResponsesAPIResponse from /v1/responses streaming). Regression test for custom pricing on streaming responses.""" import litellm - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.llms.openai import ResponsesAPIResponse custom_model_id = "gpt-5-custom-pricing" @@ -256,6 +261,121 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +class TestGetRouterModelId: + """Tests for the get_router_model_id helper method.""" + + def test_returns_id_from_litellm_metadata(self, logging_obj): + """Should extract model_info.id from litellm_metadata.""" + logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": {"id": "custom-deploy-1"}, + }, + } + assert logging_obj.get_router_model_id() == "custom-deploy-1" + + def test_returns_id_from_metadata(self, logging_obj): + """Should fall back to metadata when litellm_metadata has no model_info.""" + logging_obj.litellm_params = { + "metadata": { + "model_info": {"id": "custom-deploy-2"}, + }, + } + assert logging_obj.get_router_model_id() == "custom-deploy-2" + + def test_prefers_litellm_metadata_over_metadata(self, logging_obj): + """litellm_metadata should take priority over metadata.""" + logging_obj.litellm_params = { + "litellm_metadata": { + "model_info": {"id": "from-litellm-meta"}, + }, + "metadata": { + "model_info": {"id": "from-meta"}, + }, + } + assert logging_obj.get_router_model_id() == "from-litellm-meta" + + def test_returns_none_when_no_model_info(self, logging_obj): + """Should return None when no model_info is present.""" + logging_obj.litellm_params = {"api_base": ""} + assert logging_obj.get_router_model_id() is None + + def test_returns_none_when_no_litellm_params(self): + """Should return None when litellm_params is not set.""" + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj + + obj = LiteLLMLoggingObj( + model="test", + messages=[], + stream=False, + call_type="completion", + start_time=time.time(), + litellm_call_id="x", + function_id="x", + ) + # litellm_params exists but is empty by default + assert obj.get_router_model_id() is None + + +class TestAnthropicPassthroughCustomPricing: + """Verify the Anthropic pass-through handler forwards custom pricing.""" + + def test_completion_cost_receives_custom_pricing_args(self): + """_create_anthropic_response_logging_payload should pass + custom_pricing and router_model_id to litellm.completion_cost + when the logging object carries custom pricing in model_info.""" + from unittest.mock import patch + + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import \ + AnthropicPassthroughLoggingHandler + + logging_obj = LiteLLMLoggingObj( + model="claude-sonnet-4-20250514", + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="test-456", + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model="claude-sonnet-4-20250514", + user="", + optional_params={}, + litellm_params={ + "api_base": "", + "litellm_metadata": { + "model_info": { + "id": "claude-custom-pricing", + "input_cost_per_token": 0.5, + "output_cost_per_token": 1.5, + }, + }, + }, + ) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + mock_response = ModelResponse() + mock_response.usage = {"prompt_tokens": 10, "completion_tokens": 5} # type: ignore + + with patch("litellm.completion_cost", return_value=42.0) as mock_cost: + AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="claude-sonnet-4-20250514", + kwargs={}, + start_time=time.time(), + end_time=time.time(), + logging_obj=logging_obj, + ) + + mock_cost.assert_called_once() + call_kwargs = mock_cost.call_args + assert call_kwargs.kwargs.get("custom_pricing") is True + assert call_kwargs.kwargs.get("router_model_id") == "claude-custom-pricing" + + class TestUpdateFromKwargs: """Tests for the update_from_kwargs convenience wrapper.""" @@ -321,9 +441,8 @@ class TestUpdateFromKwargs: def test_custom_pricing_detected_via_litellm_metadata(self, logging_obj): """Custom pricing in litellm_metadata.model_info should set custom_pricing flag.""" - from litellm.litellm_core_utils.litellm_logging import ( - use_custom_pricing_for_model, - ) + from litellm.litellm_core_utils.litellm_logging import \ + use_custom_pricing_for_model lm_meta = { "model_info": { @@ -382,7 +501,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch): monkeypatch.setenv("DD_SITE", "us5.datadoghq.com") from litellm.integrations.datadog.datadog import DataDogLogger - from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger + from litellm.integrations.datadog.datadog_llm_obs import \ + DataDogLLMObsLogger from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() @@ -423,7 +543,8 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch): ) # no trailing slash on purpose # Import after env vars are set (important if module-level caching exists) - from litellm.integrations.opentelemetry import OpenTelemetry # logger class + from litellm.integrations.opentelemetry import \ + OpenTelemetry # logger class from litellm.litellm_core_utils import litellm_logging as logging_module logging_module._in_memory_loggers.clear() @@ -752,7 +873,8 @@ def test_success_handler_runs_guardrail_logging_hook_when_enabled(logging_obj): def test_get_user_agent_tags(): - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup tags = StandardLoggingPayloadSetup._get_user_agent_tags( proxy_server_request={ @@ -767,7 +889,8 @@ def test_get_user_agent_tags(): def test_get_request_tags(): - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup tags = StandardLoggingPayloadSetup._get_request_tags( litellm_params={"metadata": {"tags": ["test-tag"]}}, @@ -794,7 +917,8 @@ def test_get_request_tags_from_metadata_and_litellm_metadata(): 4. No tags in either 5. None values for metadata/litellm_metadata """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: Tags in metadata only tags = StandardLoggingPayloadSetup._get_request_tags( @@ -875,7 +999,8 @@ def test_get_request_tags_does_not_mutate_original_tags(): would cause User-Agent tags to be duplicated because the function was mutating the original tags list instead of creating a copy. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create metadata with original tags original_tags = ["custom-tag-1", "custom-tag-2"] @@ -935,7 +1060,8 @@ def test_get_request_tags_does_not_mutate_original_tags(): def test_get_extra_header_tags(): """Test the _get_extra_header_tags method with various scenarios.""" import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Store original value to restore later original_extra_headers = getattr(litellm, "extra_spend_tag_headers", None) @@ -1156,7 +1282,8 @@ async def test_e2e_generate_cold_storage_object_key_successful(): from datetime import datetime, timezone from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1198,7 +1325,8 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path() from datetime import datetime, timezone from unittest.mock import MagicMock, patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1249,7 +1377,8 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path(): from datetime import datetime, timezone from unittest.mock import MagicMock, patch - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1296,7 +1425,8 @@ async def test_e2e_generate_cold_storage_object_key_not_configured(): from unittest.mock import patch import litellm - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test data start_time = datetime(2025, 1, 15, 10, 30, 45, 123456, timezone.utc) @@ -1320,7 +1450,8 @@ def test_get_final_response_obj_with_empty_response_obj_and_list_init(): When response_obj is empty (falsy), the method should return init_response_obj if it's a list. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Create test objects class TestObject1: @@ -1356,7 +1487,8 @@ def test_get_usage_as_dict(): """ Test get_usage_as_dict returns usage as plain dict from response_obj or combined_usage_object. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup from litellm.types.utils import Usage # Test case 1: None response_obj returns empty usage dict @@ -1394,7 +1526,8 @@ def test_append_system_prompt_messages(): """ Test append_system_prompt_messages prepends system message from kwargs to messages list. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: system in kwargs with existing messages kwargs = {"system": "You are a helpful assistant"} @@ -1465,7 +1598,8 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a pass-through endpoint @@ -1546,7 +1680,8 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a pass-through endpoint @@ -1622,7 +1757,8 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_ from datetime import datetime from unittest.mock import patch - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import StandardPassThroughResponseObject # Create a logging object for a streaming pass-through endpoint @@ -1678,7 +1814,8 @@ def test_get_error_information_error_code_priority(): Test get_error_information prioritizes 'code' attribute over 'status_code' attribute and handles edge cases like empty strings and "None" string values. """ - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.litellm_core_utils.litellm_logging import \ + StandardLoggingPayloadSetup # Test case 1: Exception with 'code' attribute (ProxyException style) class ProxyException(Exception): @@ -1871,7 +2008,8 @@ async def test_async_success_handler_preserves_response_cost_for_pass_through_en by pass-through handlers (Gemini/Vertex).""" from datetime import datetime - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import \ + Logging as LiteLLMLoggingObj from litellm.types.utils import ModelResponse, Usage logging_obj = LiteLLMLoggingObj( From f7803d2d6d337d94faf34bac82d9441b52507c2f Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 21:21:07 -0700 Subject: [PATCH 39/57] chore: regenerate poetry.lock to unblock CI (pyproject.toml content hash drift) --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e3b083d778..dc25864442 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8018,4 +8018,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "eda34dfd8b35474beffee18893d6782c7b3d0d3d2c610f66237eb97176f43527" +content-hash = "2cf958f1a04fd5f1ab0e5cfc33bdbf441b518ed6c82d0f2546bf64cd3d2f89be" From 08f0cbc2e939c6456feb2f7ef8edf118644748fa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 18 Mar 2026 21:36:39 -0700 Subject: [PATCH 40/57] fix: address greptile feedback --- litellm/litellm_core_utils/litellm_logging.py | 266 +++++++++++------- litellm/responses/streaming_iterator.py | 73 +++-- ...t_base_responses_api_streaming_iterator.py | 156 +++++++++- 3 files changed, 369 insertions(+), 126 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5e34a4c992..27cef85818 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,28 +12,47 @@ import time import traceback from datetime import datetime as dt_object from functools import lru_cache -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, - Optional, Tuple, Type, Union, cast) +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Tuple, + Type, + Union, + cast, +) from httpx import Response from pydantic import BaseModel import litellm -from litellm import (_custom_logger_compatible_callbacks_literal, json_logs, - log_raw_request_response, turn_off_message_logging) +from litellm import ( + _custom_logger_compatible_callbacks_literal, + json_logs, + log_raw_request_response, + turn_off_message_logging, +) from litellm._logging import _is_debugging_on, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler -from litellm.constants import (DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - SENTRY_DENYLIST, SENTRY_PII_DENYLIST) -from litellm.cost_calculator import (RealtimeAPITokenUsageProcessor, - _select_model_name_for_cost_calc) +from litellm.constants import ( + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + SENTRY_DENYLIST, + SENTRY_PII_DENYLIST, +) +from litellm.cost_calculator import ( + RealtimeAPITokenUsageProcessor, + _select_model_name_for_cost_calc, +) from litellm.integrations.agentops import AgentOps -from litellm.integrations.anthropic_cache_control_hook import \ - AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger @@ -42,48 +61,72 @@ from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params -from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import \ - StandardBuiltInToolCostTracking -from litellm.litellm_core_utils.logging_utils import \ - truncate_base64_in_messages +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, - redact_message_input_output_from_logging) + redact_message_input_output_from_logging, +) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.llms.openai import (AllMessageValues, Batch, FineTuningJob, - HttpxBinaryResponseContent, - OpenAIFileObject, - OpenAIModerationResponse, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponseFailedEvent, - ResponseIncompleteEvent, - ResponsesAPIResponse) +from litellm.types.llms.openai import ( + AllMessageValues, + Batch, + FineTuningJob, + HttpxBinaryResponseContent, + OpenAIFileObject, + OpenAIModerationResponse, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, +) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( - CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, - CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, - EmbeddingResponse, GuardrailStatus, ImageResponse, LiteLLMBatch, - LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, ModelResponse, - ModelResponseStream, RawRequestTypedDict, StandardBuiltInToolsParams, - StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, - StandardLoggingHiddenParams, StandardLoggingMCPToolCall, - StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, - StandardLoggingModelInformation, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, StandardLoggingPayloadStatus, + CachingDetails, + CallTypes, + CostBreakdown, + CostResponseTypes, + CustomPricingLiteLLMParams, + DynamicPromptManagementParamLiteral, + EmbeddingResponse, + GuardrailStatus, + ImageResponse, + LiteLLMBatch, + LiteLLMLoggingBaseClass, + LiteLLMRealtimeStreamLoggingObject, + ModelResponse, + ModelResponseStream, + RawRequestTypedDict, + StandardBuiltInToolsParams, + StandardCallbackDynamicParams, + StandardLoggingAdditionalHeaders, + StandardLoggingHiddenParams, + StandardLoggingMCPToolCall, + StandardLoggingMetadata, + StandardLoggingModelCostFailureDebugInformation, + StandardLoggingModelInformation, + StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, + StandardLoggingPayloadStatus, StandardLoggingPayloadStatusFields, - StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, - TextCompletionResponse, TranscriptionResponse, Usage) + StandardLoggingPromptManagementMetadata, + StandardLoggingVectorStoreRequest, + TextCompletionResponse, + TranscriptionResponse, + Usage, +) from litellm.types.videos.main import VideoObject -from litellm.utils import (_get_base_model_from_metadata, executor, - print_verbose) +from litellm.utils import _get_base_model_from_metadata, executor, print_verbose from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -105,8 +148,7 @@ from ..integrations.humanloop import HumanloopLogger from ..integrations.lago import LagoLogger from ..integrations.langfuse.langfuse import LangFuseLogger from ..integrations.langfuse.langfuse_handler import LangFuseHandler -from ..integrations.langfuse.langfuse_prompt_management import \ - LangfusePromptManagement +from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement from ..integrations.langsmith import LangsmithLogger from ..integrations.litellm_agent import LiteLLMAgentModelResolver from ..integrations.literal_ai import LiteralAILogger @@ -121,30 +163,34 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers -from .initialize_dynamic_callback_params import \ - initialize_standard_callback_dynamic_params as \ - _initialize_standard_callback_dynamic_params +from .initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, +) from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from litellm.llms.base_llm.passthrough.transformation import \ - BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: - from litellm_enterprise.enterprise_callbacks.callback_controls import \ - EnterpriseCallbackControls - from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import \ - PagerDutyAlerting - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ - ResendEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ - SendGridEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ - SMTPEmailLogger - from litellm_enterprise.litellm_core_utils.litellm_logging import \ - StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup + from litellm_enterprise.enterprise_callbacks.callback_controls import ( + EnterpriseCallbackControls, + ) + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( + PagerDutyAlerting, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( + SendGridEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) + from litellm_enterprise.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup, + ) - from litellm.integrations.generic_api.generic_api_callback import \ - GenericAPILogger + from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger EnterpriseStandardLoggingPayloadSetupVAR: Optional[ Type[EnterpriseStandardLoggingPayloadSetup] @@ -1723,8 +1769,9 @@ class Logging(LiteLLMLoggingBaseClass): ) standard_logging_payload["response"] = response_dict elif isinstance(result, TranscriptionResponse): - from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import \ - TranscriptionUsageObjectTransformation + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, + ) result = result.model_copy() transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore @@ -2407,8 +2454,9 @@ class Logging(LiteLLMLoggingBaseClass): ): # polling job will query these frequently, don't spam db logs return - from litellm.proxy.openai_files_endpoints.common_utils import \ - _is_base64_encoded_unified_file_id + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + ) # check if file id is a unified file id is_base64_unified_file_id = _is_base64_encoded_unified_file_id(result.id) @@ -3305,7 +3353,6 @@ class Logging(LiteLLMLoggingBaseClass): return result.response else: return None - return None def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: """ @@ -3551,8 +3598,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "s3": s3Logger = S3Logger() elif callback == "wandb": - from litellm.integrations.weights_biases import \ - WeightsBiasesLogger + from litellm.integrations.weights_biases import WeightsBiasesLogger weightsBiasesLogger = WeightsBiasesLogger() elif callback == "logfire": @@ -3616,8 +3662,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_posthog_logger) return _posthog_logger # type: ignore elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -3738,7 +3783,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _opik_logger # type: ignore elif logging_integration == "arize": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_config = ArizeLogger.get_arize_config() if arize_config.endpoint is None: @@ -3765,7 +3812,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_phoenix_config = ArizePhoenixLogger.get_arize_phoenix_config() otel_config = OpenTelemetryConfig( @@ -3819,7 +3868,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "levo": from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) levo_config = LevoLogger.get_levo_config() otel_config = OpenTelemetryConfig( @@ -3868,8 +3919,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(galileo_logger) return galileo_logger # type: ignore elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -3889,8 +3939,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(focus_logger) return focus_logger # type: ignore elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -3910,7 +3959,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) logfire_base_url = os.getenv( "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev" @@ -3928,8 +3979,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): @@ -3951,8 +4003,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -3978,7 +4031,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 raise ValueError("LANGTRACE_API_KEY not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) otel_config = OpenTelemetryConfig( exporter="otlp_http", @@ -4014,8 +4069,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": - from litellm.integrations.langfuse.langfuse_otel import \ - LangfuseOtelLogger + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger for callback in _in_memory_loggers: if ( @@ -4033,7 +4087,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "weave_otel": from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( - WeaveOtelLogger, get_weave_otel_config) + WeaveOtelLogger, + get_weave_otel_config, + ) weave_otel_config = get_weave_otel_config() @@ -4069,8 +4125,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(anthropic_cache_control_hook) return anthropic_cache_control_hook # type: ignore elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -4130,8 +4187,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dotprompt_logger) return dotprompt_logger # type: ignore elif logging_integration == "bitbucket": - from litellm.integrations.bitbucket.bitbucket_prompt_manager import \ - BitBucketPromptManager + from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( + BitBucketPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, BitBucketPromptManager): @@ -4148,8 +4206,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(bitbucket_logger) return bitbucket_logger # type: ignore elif logging_integration == "gitlab": - from litellm.integrations.gitlab.gitlab_prompt_manager import \ - GitLabPromptManager + from litellm.integrations.gitlab.gitlab_prompt_manager import ( + GitLabPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, GitLabPromptManager): @@ -4237,8 +4296,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, OpenMeterLogger): return callback elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -4248,8 +4306,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, GalileoObserve): return callback elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -4263,8 +4320,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 ): # exact match; exclude subclasses like VantageLogger return callback elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -4364,15 +4420,17 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4404,8 +4462,9 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, AnthropicCacheControlHook): return callback elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -5577,6 +5636,7 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal + # used for unit testing from typing import Any, Dict, List, Optional, Union diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a6a1074067..8a91368dd6 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,29 +8,31 @@ from typing import Any, Dict, List, Optional import httpx import litellm -from litellm.constants import (LITELLM_MAX_STREAMING_DURATION_SECONDS, - STREAM_SSE_DONE_STRING) +from litellm.constants import ( + LITELLM_MAX_STREAMING_DURATION_SECONDS, + STREAM_SSE_DONE_STRING, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_response_utils.get_api_base import \ - get_api_base -from litellm.litellm_core_utils.llm_response_utils.response_metadata import \ - update_response_metadata +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, +) from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.responses.transformation import \ - BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import (OutputTextDeltaEvent, ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, - ResponsesAPIStreamingResponse) +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse, +) from litellm.types.utils import CallTypes -from litellm.utils import (CustomStreamWrapper, - async_post_call_success_deployment_hook) +from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook class BaseResponsesAPIStreamingIterator: @@ -198,10 +200,12 @@ class BaseResponsesAPIStreamingIterator: if cost is not None: setattr(usage_obj, "cost", cost) except Exception: - # If cost calculation fails, continue without cost pass - self._handle_logging_completed_response() + if _chunk_type == ResponsesAPIStreamEvents.RESPONSE_FAILED: + self._handle_logging_failed_response() + else: + self._handle_logging_completed_response() return openai_responses_api_chunk @@ -219,6 +223,32 @@ class BaseResponsesAPIStreamingIterator: """Base implementation - should be overridden by subclasses""" pass + def _handle_logging_failed_response(self): + """ + Handle logging for RESPONSE_FAILED events by routing to failure handlers. + + Unlike _handle_logging_completed_response (which calls success handlers), + this constructs an exception from the response error and routes to + async_failure_handler / failure_handler so logging integrations correctly + record the call as failed. + """ + response_obj = ( + getattr(self.completed_response, "response", None) + if self.completed_response + else None + ) + error_info = getattr(response_obj, "error", None) if response_obj else None + error_message = "Response failed" + if isinstance(error_info, dict): + error_message = error_info.get("message", str(error_info)) + exception = litellm.APIError( + status_code=500, + message=error_message, + llm_provider=self.custom_llm_provider or "", + model=self.model or "", + ) + self._handle_failure(exception) + async def _call_post_streaming_deployment_hook(self, chunk): """ Allow callbacks to modify streaming chunks before returning (parity with chat). @@ -697,8 +727,7 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): # --------------------------------------------------------------------------- from litellm._logging import verbose_logger -from litellm.litellm_core_utils.thread_pool_executor import \ - executor as _ws_executor +from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor RESPONSES_WS_LOGGED_EVENT_TYPES = [ "response.created", diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 860445d875..e9181d810e 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -30,9 +30,11 @@ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterat from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, - OutputTextDeltaEvent + OutputTextDeltaEvent, ) @@ -429,3 +431,155 @@ class TestBaseResponsesAPIStreamingIterator: mock_logging_obj.async_failure_handler.assert_not_called() mock_logging_obj.failure_handler.assert_not_called() + def test_process_chunk_response_failed_calls_failure_handler(self): + """ + Test that a RESPONSE_FAILED event routes to failure handlers, + not success handlers. Failed responses represent genuine LLM-level + errors and should be logged as failures. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_failed_123" + mock_responses_api_response.error = { + "type": "server_error", + "message": "The model encountered an error", + } + mock_responses_api_response.usage = None + + mock_failed_event = Mock(spec=ResponseFailedEvent) + mock_failed_event.type = ResponsesAPIStreamEvents.RESPONSE_FAILED + mock_failed_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_failed_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.failed", + "response": { + "id": "resp_failed_123", + "error": { + "type": "server_error", + "message": "The model encountered an error", + }, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "litellm.responses.streaming_iterator.run_async_function" + ) as mock_run_async, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_FAILED + assert iterator.completed_response == result + + # Failure handler should have been called via _handle_failure + mock_run_async.assert_called_once() + call_kwargs = mock_run_async.call_args + assert ( + call_kwargs[1]["async_function"] + == mock_logging_obj.async_failure_handler + ) + + mock_executor.submit.assert_called_once() + submit_args = mock_executor.submit.call_args + assert submit_args[0][0] == mock_logging_obj.failure_handler + + def test_process_chunk_response_incomplete_calls_success_handler(self): + """ + Test that a RESPONSE_INCOMPLETE event routes to success handlers. + Incomplete responses (e.g. max_output_tokens reached) are still valid + responses with usage data — analogous to finish_reason='length' in chat. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_incomplete_123" + mock_responses_api_response.incomplete_details = { + "reason": "max_output_tokens" + } + mock_responses_api_response.usage = None + + mock_incomplete_event = Mock(spec=ResponseIncompleteEvent) + mock_incomplete_event.type = ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + mock_incomplete_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_incomplete_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.incomplete", + "response": { + "id": "resp_incomplete_123", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "asyncio.create_task" + ) as mock_create_task, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + assert iterator.completed_response == result + + # Success handler should have been called (via _handle_logging_completed_response) + mock_create_task.assert_called_once() + mock_executor.submit.assert_called_once() + + # Failure handlers should NOT have been called + mock_logging_obj.async_failure_handler.assert_not_called() + mock_logging_obj.failure_handler.assert_not_called() + From df38fbcc973b269d70d6c6c6891d444d70e9f04d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 19 Mar 2026 04:47:30 +0000 Subject: [PATCH 41/57] docs: add Contributing to Guardrails section to Guardrail Providers sidebar - Add 'Contributing to Guardrails' category with links to: - Generic Guardrail API (integrate without PR) - Adding a New Guardrail Integration tutorial - Adding Guardrail Support to Endpoints - Add 'Team Bring-Your-Own Guardrails' link for team BYOG workflow These docs existed but were only accessible from the 'LiteLLM AI Gateway' sidebar. Now they're also accessible when browsing the 'Guardrail Providers' section. Co-authored-by: Krish Dholakia --- docs/my-website/sidebars.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 7db72da276..4c0471fb8f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -48,6 +48,20 @@ const sidebars = { slug: "/guardrail_providers" }, items: [ + { + type: "category", + label: "Contributing to Guardrails", + items: [ + "adding_provider/generic_guardrail_api", + "adding_provider/simple_guardrail_tutorial", + "adding_provider/adding_guardrail_support", + ] + }, + { + type: "doc", + id: "proxy/guardrails/team_based_guardrails", + label: "Team Bring-Your-Own Guardrails", + }, ...[ "proxy/guardrails/qualifire", "proxy/guardrails/aim_security", From dab8721ba316a6b859d8636c80bc36b94f32a767 Mon Sep 17 00:00:00 2001 From: joereyna Date: Wed, 18 Mar 2026 22:57:38 -0700 Subject: [PATCH 42/57] chore: apply black formatting to fix lint CI --- litellm/litellm_core_utils/litellm_logging.py | 7 ++----- litellm/proxy/management_endpoints/team_endpoints.py | 4 +++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4e63dd7076..1b612c7091 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1460,9 +1460,7 @@ class Logging(LiteLLMLoggingBaseClass): # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. if router_model_id is None and hasattr(self, "litellm_params"): for metadata_key in ("litellm_metadata", "metadata"): - _metadata: dict = ( - self.litellm_params.get(metadata_key, {}) or {} - ) + _metadata: dict = self.litellm_params.get(metadata_key, {}) or {} _model_info: dict = _metadata.get("model_info", {}) or {} _model_id = _model_info.get("id") if _model_id is not None: @@ -2972,8 +2970,7 @@ class Logging(LiteLLMLoggingBaseClass): if ( isinstance(callback, CustomLogger) and is_sync_request - and self.call_type - != CallTypes.pass_through.value + and self.call_type != CallTypes.pass_through.value ): # custom logger class callback.log_failure_event( start_time=start_time, diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index b033d3fce0..3d4488b8a7 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -3334,7 +3334,9 @@ def _convert_teams_to_response_models( use_deleted_table: bool, ) -> List[Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable]]: """Convert raw Prisma team rows to response models.""" - team_list: List[Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable]] = [] + team_list: List[ + Union[TeamListItem, LiteLLM_TeamTable, LiteLLM_DeletedTeamTable] + ] = [] for team in teams: try: team_dict = team.model_dump() From 532e0d13df3b3e4532bc805d02083fa7f01980dc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 15:57:03 +0530 Subject: [PATCH 43/57] feat(proxy): use AZURE_DEFAULT_API_VERSION for proxy --api_version default Aligns proxy default with litellm.AZURE_DEFAULT_API_VERSION (2025-02-01-preview) so Azure response_format + json_schema works without tools fallback. Made-with: Cursor --- litellm/proxy/proxy_cli.py | 3 +- tests/test_litellm/proxy/test_proxy_cli.py | 41 ++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 97d5de0d53..c638e29426 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -12,6 +12,7 @@ import click import httpx from dotenv import load_dotenv +import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY from litellm.secret_managers.main import get_secret_bool @@ -387,7 +388,7 @@ class ProxyInitializationHelpers: @click.option("--api_base", default=None, help="API base URL.") @click.option( "--api_version", - default="2024-07-01-preview", + default=litellm.AZURE_DEFAULT_API_VERSION, help="For azure - pass in the api version.", ) @click.option( diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index c5d6c45f9a..349fe76ed7 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -280,6 +280,47 @@ class TestProxyInitializationHelpers: assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" mock_uvicorn_run.assert_called_once() + @patch("uvicorn.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_proxy_default_api_version_uses_azure_default( + self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run + ): + """Proxy default api_version should match litellm.AZURE_DEFAULT_API_VERSION for consistency.""" + from click.testing import CliRunner + + import litellm + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")} + with patch.dict(os.environ, clean_env, clear=True), patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args: + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + result = runner.invoke(run_server, ["--local", "--skip_server_startup"]) + assert result.exit_code == 0, f"exit_code={result.exit_code}, output={result.output}" + mock_proxy_module.save_worker_config.assert_called_once() + call_kwargs = mock_proxy_module.save_worker_config.call_args[1] + assert call_kwargs["api_version"] == litellm.AZURE_DEFAULT_API_VERSION + @patch("uvicorn.run") @patch("builtins.print") def test_keepalive_timeout_flag(self, mock_print, mock_uvicorn_run): From 067dab42e6fc8a455434f05926cfae88a6638b47 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 16:16:23 +0530 Subject: [PATCH 44/57] refactor: reduce statement count in langsmith and anthropic methods - Extract helper methods in langsmith._prepare_log_data to reduce from 51 to <50 statements - Extract helper methods in anthropic.transform_parsed_response to reduce from 57 to <50 statements - Fixes PLR0915 linter errors - All existing tests pass (10 langsmith tests, 126 anthropic tests) Made-with: Cursor --- litellm/integrations/langsmith.py | 143 +++++------ litellm/llms/anthropic/chat/transformation.py | 243 ++++++++++-------- 2 files changed, 196 insertions(+), 190 deletions(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index ef2d30bb26..479b5027ef 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -5,7 +5,6 @@ import os import random import traceback import types -from litellm._uuid import uuid from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -14,10 +13,11 @@ from pydantic import BaseModel # type: ignore import litellm from litellm._logging import verbose_logger +from litellm._uuid import uuid from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.langsmith_mock_client import ( - should_use_langsmith_mock, create_mock_langsmith_client, + should_use_langsmith_mock, ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -110,6 +110,56 @@ class LangsmithLogger(CustomBatchLogger): LANGSMITH_TENANT_ID=_credentials_tenant_id, ) + def _extract_metadata_fields( + self, metadata: dict, credentials: LangsmithCredentialsObject + ): + return { + "project_name": metadata.get("project_name", credentials["LANGSMITH_PROJECT"]), + "run_name": metadata.get("run_name", self.langsmith_default_run_name), + "run_id": metadata.get("id", metadata.get("run_id", None)), + "parent_run_id": metadata.get("parent_run_id", None), + "trace_id": metadata.get("trace_id", None), + "session_id": metadata.get("session_id", None), + "dotted_order": metadata.get("dotted_order", None), + } + + def _build_extra_metadata(self, metadata: Dict): + extra_metadata = dict(metadata) + requester_metadata = extra_metadata.get("requester_metadata") + if requester_metadata and isinstance(requester_metadata, dict): + for key in ("session_id", "thread_id", "conversation_id"): + if key in requester_metadata and key not in extra_metadata: + extra_metadata[key] = requester_metadata[key] + return extra_metadata + + def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> Dict[str, Any]: + response = payload["response"] + outputs: Dict[str, Any] + if isinstance(response, dict): + outputs = {**response} + else: + outputs = {"output": response} + outputs["usage_metadata"] = { + "input_tokens": payload.get("prompt_tokens", 0), + "output_tokens": payload.get("completion_tokens", 0), + "total_tokens": payload.get("total_tokens", 0), + "total_cost": payload.get("response_cost", 0), + } + return outputs + + def _ensure_required_ids(self, data: dict, run_id: Optional[str]): + if "id" not in data or data["id"] is None: + run_id = str(uuid.uuid4()) + data["id"] = run_id + + if "trace_id" not in data or data["trace_id"] is None: + if run_id is not None and isinstance(run_id, str): + data["trace_id"] = run_id + + if "dotted_order" not in data or data["dotted_order"] is None: + if run_id is not None and isinstance(run_id, str): + data["dotted_order"] = self.make_dot_order(run_id=run_id) + def _prepare_log_data( self, kwargs, @@ -121,56 +171,28 @@ class LangsmithLogger(CustomBatchLogger): try: _litellm_params = kwargs.get("litellm_params", {}) or {} metadata = _litellm_params.get("metadata", {}) or {} - project_name = metadata.get( - "project_name", credentials["LANGSMITH_PROJECT"] - ) - run_name = metadata.get("run_name", self.langsmith_default_run_name) - run_id = metadata.get("id", metadata.get("run_id", None)) - parent_run_id = metadata.get("parent_run_id", None) - trace_id = metadata.get("trace_id", None) - session_id = metadata.get("session_id", None) - dotted_order = metadata.get("dotted_order", None) + + fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( - f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" + f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" ) - # Ensure everything in the payload is converted to str payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object", None ) - if payload is None: raise Exception("Error logging request payload. Payload=none.") - metadata = payload[ - "metadata" - ] # ensure logged metadata is json serializable - - extra_metadata = dict(metadata) - requester_metadata = extra_metadata.get("requester_metadata") - if requester_metadata and isinstance(requester_metadata, dict): - for key in ("session_id", "thread_id", "conversation_id"): - if key in requester_metadata and key not in extra_metadata: - extra_metadata[key] = requester_metadata[key] - - outputs = payload["response"] - if isinstance(outputs, dict): - outputs = {**outputs} - else: - outputs = {"output": outputs} - outputs["usage_metadata"] = { - "input_tokens": payload.get("prompt_tokens", 0), - "output_tokens": payload.get("completion_tokens", 0), - "total_tokens": payload.get("total_tokens", 0), - "total_cost": payload.get("response_cost", 0), - } + metadata = payload["metadata"] + extra_metadata = self._build_extra_metadata(dict(metadata)) + outputs = self._build_outputs_with_usage(payload) data = { - "name": run_name, - "run_type": "llm", # this should always be llm, since litellm always logs llm calls. Langsmith allow us to log "chain" + "name": fields["run_name"], + "run_type": "llm", "inputs": payload, "outputs": outputs, - "session_name": project_name, + "session_name": fields["project_name"], "start_time": payload["startTime"], "end_time": payload["endTime"], "tags": payload["request_tags"], @@ -180,46 +202,13 @@ class LangsmithLogger(CustomBatchLogger): if payload["error_str"] is not None and payload["status"] == "failure": data["error"] = payload["error_str"] - if run_id: - data["id"] = run_id - - if parent_run_id: - data["parent_run_id"] = parent_run_id - - if trace_id: - data["trace_id"] = trace_id - - if session_id: - data["session_id"] = session_id - - if dotted_order: - data["dotted_order"] = dotted_order - - run_id: Optional[str] = data.get("id") # type: ignore - if "id" not in data or data["id"] is None: - """ - for /batch langsmith requires id, trace_id and dotted_order passed as params - """ - run_id = str(uuid.uuid4()) - - data["id"] = run_id - - if ( - "trace_id" not in data - or data["trace_id"] is None - and (run_id is not None and isinstance(run_id, str)) - ): - data["trace_id"] = run_id - - if ( - "dotted_order" not in data - or data["dotted_order"] is None - and (run_id is not None and isinstance(run_id, str)) - ): - data["dotted_order"] = self.make_dot_order(run_id=run_id) # type: ignore + for key in ("id", "parent_run_id", "trace_id", "session_id", "dotted_order"): + field_key = "run_id" if key == "id" else key + if fields[field_key]: + data[key] = fields[field_key] + self._ensure_required_ids(data, fields["run_id"]) verbose_logger.debug("Langsmith Logging data on langsmith: %s", data) - return data except Exception: raise diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index f2fc4601cb..808b68fefd 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -50,6 +50,10 @@ from litellm.types.llms.openai import ( OpenAIMcpServerTool, OpenAIWebSearchOptions, ) +from litellm.types.responses.main import ( + OutputCodeInterpreterCall, + build_code_interpreter_log_outputs, +) from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -59,10 +63,6 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, ServerToolUse, ) -from litellm.types.responses.main import ( - OutputCodeInterpreterCall, - build_code_interpreter_log_outputs, -) from litellm.utils import ( ModelResponse, Usage, @@ -1684,6 +1684,85 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage + def _build_code_by_id_map(self, tool_calls: List[ChatCompletionToolCallChunk]) -> Dict[str, str]: + code_by_id: Dict[str, str] = {} + for tc in tool_calls: + try: + args = json.loads(tc.get("function", {}).get("arguments", "{}")) + call_id = tc.get("id") + command = args.get("command", "") + if isinstance(call_id, str): + code_by_id[call_id] = command if isinstance(command, str) else "" + except Exception: + pass + return code_by_id + + def _build_code_interpreter_results( + self, tool_results: List[Any], code_by_id: Dict[str, str], container_id: Optional[str] + ) -> List[OutputCodeInterpreterCall]: + code_interpreter_results = [] + for tr in tool_results: + if tr.get("type") != "bash_code_execution_tool_result": + continue + call_id = tr.get("tool_use_id", "") + content = tr.get("content", {}) + log_outputs = build_code_interpreter_log_outputs(content) + code_interpreter_results.append( + OutputCodeInterpreterCall( + type="code_interpreter_call", + id=call_id, + code=code_by_id.get(call_id, ""), + container_id=container_id, + status="completed", + outputs=log_outputs, + ) + ) + return code_interpreter_results + + def _build_provider_specific_fields( + self, + completion_response: dict, + citations: Optional[List[Any]], + thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]], + web_search_results: Optional[List[Any]], + tool_results: Optional[List[Any]], + compaction_blocks: Optional[List[Any]], + tool_calls: List[ChatCompletionToolCallChunk], + ) -> Dict[str, Any]: + provider_specific_fields: Dict[str, Any] = { + "citations": citations, + "thinking_blocks": thinking_blocks, + } + + context_management = completion_response.get("context_management") + if context_management is not None: + provider_specific_fields["context_management"] = context_management + + if web_search_results is not None: + provider_specific_fields["web_search_results"] = web_search_results + + if tool_results is not None: + provider_specific_fields["tool_results"] = tool_results + container_id = ( + completion_response.get("container", {}).get("id") + if isinstance(completion_response.get("container"), dict) + else None + ) + code_by_id = self._build_code_by_id_map(tool_calls) + code_interpreter_results = self._build_code_interpreter_results( + tool_results, code_by_id, container_id + ) + provider_specific_fields["code_interpreter_results"] = code_interpreter_results + + container = completion_response.get("container") + if container is not None: + provider_specific_fields["container"] = container + + if compaction_blocks is not None: + provider_specific_fields["compaction_blocks"] = compaction_blocks + + return provider_specific_fields + def transform_parsed_response( self, completion_response: dict, @@ -1704,128 +1783,66 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): status_code=raw_response.status_code, headers=response_headers, ) - else: - text_content = "" - citations: Optional[List[Any]] = None - thinking_blocks: Optional[ - List[ - Union[ - ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock - ] - ] - ] = None - reasoning_content: Optional[str] = None - tool_calls: List[ChatCompletionToolCallChunk] = [] - ( - text_content, - citations, - thinking_blocks, - reasoning_content, - tool_calls, - web_search_results, - tool_results, - compaction_blocks, - ) = self.extract_response_content(completion_response=completion_response) + ( + text_content, + citations, + thinking_blocks, + reasoning_content, + tool_calls, + web_search_results, + tool_results, + compaction_blocks, + ) = self.extract_response_content(completion_response=completion_response) - if ( - prefix_prompt is not None - and not text_content.startswith(prefix_prompt) - and not litellm.disable_add_prefix_to_prompt - ): - text_content = prefix_prompt + text_content + if ( + prefix_prompt is not None + and not text_content.startswith(prefix_prompt) + and not litellm.disable_add_prefix_to_prompt + ): + text_content = prefix_prompt + text_content - context_management: Optional[Dict] = completion_response.get( - "context_management" - ) + provider_specific_fields = self._build_provider_specific_fields( + completion_response, + citations, + thinking_blocks, + web_search_results, + tool_results, + compaction_blocks, + tool_calls, + ) - container: Optional[Dict] = completion_response.get("container") + _message = litellm.Message( + tool_calls=tool_calls, + content=text_content or None, + provider_specific_fields=provider_specific_fields, + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, + ) + _message.provider_specific_fields = provider_specific_fields - provider_specific_fields: Dict[str, Any] = { - "citations": citations, - "thinking_blocks": thinking_blocks, - } - if context_management is not None: - provider_specific_fields["context_management"] = context_management - if web_search_results is not None: - provider_specific_fields["web_search_results"] = web_search_results - if tool_results is not None: - provider_specific_fields["tool_results"] = tool_results - # Convert to provider-neutral OutputCodeInterpreterCall objects - # so the Responses API layer can use them without Anthropic-specific knowledge. - container_id = ( - completion_response.get("container", {}).get("id") - if isinstance(completion_response.get("container"), dict) - else None - ) - code_by_id: Dict[str, str] = {} - for tc in tool_calls: - try: - args = json.loads(tc.get("function", {}).get("arguments", "{}")) - code_by_id[tc.get("id", "")] = args.get("command", "") - except Exception: - pass - code_interpreter_results = [] - for tr in tool_results: - if tr.get("type") != "bash_code_execution_tool_result": - continue - call_id = tr.get("tool_use_id", "") - content = tr.get("content", {}) - log_outputs = build_code_interpreter_log_outputs(content) - code_interpreter_results.append( - OutputCodeInterpreterCall( - type="code_interpreter_call", - id=call_id, - code=code_by_id.get(call_id, ""), - container_id=container_id, - status="completed", - outputs=log_outputs, - ) - ) - provider_specific_fields["code_interpreter_results"] = ( - code_interpreter_results - ) - if container is not None: - provider_specific_fields["container"] = container - if compaction_blocks is not None: - provider_specific_fields["compaction_blocks"] = compaction_blocks + json_mode_message = self._transform_response_for_json_mode( + json_mode=json_mode, + tool_calls=tool_calls, + ) + if json_mode_message is not None: + completion_response["stop_reason"] = "stop" + _message = json_mode_message - _message = litellm.Message( - tool_calls=tool_calls, - content=text_content or None, - provider_specific_fields=provider_specific_fields, - thinking_blocks=thinking_blocks, - reasoning_content=reasoning_content, - ) - _message.provider_specific_fields = provider_specific_fields + model_response.choices[0].message = _message + model_response._hidden_params["original_response"] = completion_response["content"] + model_response.choices[0].finish_reason = cast( + OpenAIChatCompletionFinishReason, + map_finish_reason(completion_response["stop_reason"]), + ) - ## HANDLE JSON MODE - anthropic returns single function call - json_mode_message = self._transform_response_for_json_mode( - json_mode=json_mode, - tool_calls=tool_calls, - ) - if json_mode_message is not None: - completion_response["stop_reason"] = "stop" - _message = json_mode_message - - model_response.choices[0].message = _message # type: ignore - model_response._hidden_params["original_response"] = completion_response[ - "content" - ] # allow user to access raw anthropic tool calling response - - model_response.choices[0].finish_reason = cast( - OpenAIChatCompletionFinishReason, - map_finish_reason(completion_response["stop_reason"]), - ) - - ## CALCULATING USAGE usage = self.calculate_usage( usage_object=completion_response["usage"], reasoning_content=reasoning_content, completion_response=completion_response, speed=speed, ) - setattr(model_response, "usage", usage) # type: ignore + setattr(model_response, "usage", usage) model_response.created = int(time.time()) model_response.model = completion_response["model"] From b9564834e6eea59a85666685e78a31febbbbbcca Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 19 Mar 2026 16:18:06 +0530 Subject: [PATCH 45/57] Fix mypy errors --- litellm/llms/anthropic/chat/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 7dce72f1e8..a2389f4429 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -897,7 +897,9 @@ class ModelResponseIterator: args = "" for block in self.content_blocks: if block["delta"]["type"] == "input_json_delta": - args += block["delta"].get("partial_json", "") + partial_json = block["delta"].get("partial_json") + if isinstance(partial_json, str): + args += partial_json if args: try: self._server_tool_inputs[ From 81dadb698a5984a4bf825903b3384927d12d54bc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 19 Mar 2026 10:20:35 -0700 Subject: [PATCH 46/57] Ishaan - March 18th changes (#24056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add DD Tracing (#24033) * feat(models): add Azure GPT-5.4 mini and nano variants (#24045) Add `azure/gpt-5.4-mini` and `azure/gpt-5.4-nano` to the model database with official pricing from Azure OpenAI: - GPT-5.4 mini: $0.75/M input, $0.075/M cached, $4.5/M output - GPT-5.4 nano: $0.20/M input, $0.02/M cached, $1.25/M output Both models support: - 1.05M input / 128K output context window - Chat, batch, and responses endpoints - Function calling, tools, vision, reasoning - Prompt caching with automatic tiered pricing Co-authored-by: Claude Opus 4.6 * Add new model pricing details for volcengine Doubao-Seed-2.0 series (#23871) Add entries for volcengine Doubao-Seed-2.0 series * fix(mcp): support refresh_token grant type in OAuth token endpoint (#23701) * fix(mcp): support refresh_token grant type in OAuth token endpoint (#23700) The .well-known/oauth-authorization-server metadata advertises refresh_token as a supported grant type, but the token endpoint rejected it with HTTP 400. This adds refresh_token grant support so MCP clients can refresh expired tokens without re-authenticating. * test(mcp): add tests for refresh_token grant type in OAuth token endpoint * fix(mcp): move code_verifier guard into authorization_code branch code_verifier is only relevant for authorization_code grants (PKCE). Move it inside the else branch so it doesn't apply to refresh_token. * fix(mcp): guard None client_secret and forward scope in token exchange - Conditionally include client_secret in form data to prevent httpx from sending the literal string "None" (applies to both authorization_code and refresh_token branches) - Forward optional scope parameter per RFC 6749 §6, allowing clients to request a subset of originally-granted scopes on refresh * fix(mcp): validate code param in authorization_code grant Guard against None code being form-encoded as literal string "None" by httpx, symmetric with the existing refresh_token guard. * docs: add incident report for guardrail logging secret exposure (#24059) Add blog post documenting the guardrail logging path exposing internal request data (e.g. Authorization headers) in spend logs and OTEL traces. Fix available in LiteLLM 1.82.3+. Made-with: Cursor * [Fix] Datadog LLM Observability tags format (env, service, version missing) (#23673) * tag fix * greptile comment * fix(ci): stabilize 6 failing CI jobs 1. mypy: remove duplicate type annotation for token_data in discoverable_endpoints.py 2. integrations tests: add parameterized to CI test deps 3. doc quality: document OTEL_IGNORE_CONTEXT_PROPAGATION env key 4. security: allowlist CVE-2026-2673, CVE-2026-3644, CVE-2026-4224 (no fix available) 5. proxy_store_model_in_db: fix missing x-litellm-call-id header on error responses 6. google tests: add --retries 3 for transient Vertex AI rate limits Co-authored-by: Ishaan Jaff * fix(streaming): handle RuntimeError during model_copy in streaming handler The race condition occurs when model_copy(deep=True) tries to deepcopy _hidden_params dict while it's being concurrently modified by logging callbacks. Fall back to shallow copy if the deep copy fails. Co-authored-by: Ishaan Jaff * fix(cost): handle non-string traffic_type in cost calculator + add retries 1. Fix AttributeError in _map_traffic_type_to_service_tier when traffic_type is an integer (cast to str before calling .upper()). This was causing pass-through vertex spend logging to fail silently. 2. Add --retries to llm_translation_testing for flaky external API calls. Co-authored-by: Ishaan Jaff --------- Co-authored-by: Emerson Gomes Co-authored-by: Claude Opus 4.6 Co-authored-by: ExMatics HydrogenC <33123710+HydrogenC@users.noreply.github.com> Co-authored-by: Jack Venberg Co-authored-by: milan-berri Co-authored-by: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Co-authored-by: Cursor Agent Co-authored-by: Ishaan Jaff --- .circleci/config.yml | 6 +- ci_cd/security_scans.sh | 3 + .../index.md | 78 ++++++ docs/my-website/docs/proxy/config_settings.md | 1 + litellm/cost_calculator.py | 2 +- litellm/integrations/datadog/datadog.py | 10 +- .../integrations/datadog/datadog_handler.py | 11 +- .../integrations/datadog/datadog_llm_obs.py | 4 +- .../litellm_core_utils/streaming_handler.py | 26 +- ...odel_prices_and_context_window_backup.json | 72 +++++ .../mcp_server/discoverable_endpoints.py | 56 +++- litellm/proxy/auth/auth_checks.py | 147 +++++----- litellm/proxy/auth/user_api_key_auth.py | 264 +++++++++--------- litellm/proxy/common_request_processing.py | 4 +- .../mcp_management_endpoints.py | 4 + model_prices_and_context_window.json | 224 +++++++++++++++ tests/logging_callback_tests/test_datadog.py | 18 +- .../datadog/test_datadog_tags_regression.py | 2 +- .../mcp_server/test_discoverable_endpoints.py | 138 +++++++++ .../test_mcp_management_endpoints.py | 52 ++++ 20 files changed, 881 insertions(+), 241 deletions(-) create mode 100644 docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md diff --git a/.circleci/config.yml b/.circleci/config.yml index 12e3cb1f6b..790efc7986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ commands: "pydantic==2.11.0" "mcp==1.25.0" "requests-mock>=1.12.1" \ "responses==0.25.7" "pytest-xdist==3.6.1" "pytest-timeout==2.2.0" \ "pytest-cov==5.0.0" "semantic_router==0.1.10" "fastapi-offline==1.7.3" \ - "a2a" + "a2a" "parameterized>=0.9.0" - setup_litellm_enterprise_pip - save_cache: paths: @@ -1115,7 +1115,7 @@ jobs: for dir in "${IGNORE_DIRS[@]}"; do IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" done - python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread + python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 no_output_timeout: 15m # Store test results @@ -1331,7 +1331,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 no_output_timeout: 15m - run: name: Rename the coverage files diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index e0f370e003..801b700f64 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -163,6 +163,9 @@ run_grype_scans() { "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code + "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet + "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image + "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image ) # Build JSON array of allowlisted CVE IDs for jq diff --git a/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md new file mode 100644 index 0000000000..71f9e3da01 --- /dev/null +++ b/docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md @@ -0,0 +1,78 @@ +--- +slug: guardrail-logging-secret-exposure-incident +title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces" +date: 2026-03-18T10:00:00 +authors: + - litellm +tags: [incident-report, security, guardrails] +hide_table_of_contents: false +--- + +**Date:** March 18, 2026 +**Duration:** Unknown +**Severity:** High +**Status:** Resolved + +## Summary + +When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials. + +This information could then propagate to logging and observability surfaces that consume guardrail metadata, including: + +- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data +- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend + +LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths. + +{/* truncate */} + +--- + +## Background + +LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry. + +When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems. + +```mermaid +flowchart TD + inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"] + storeSecrets --> guardrailRuns["3. Custom guardrail runs"] + guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"] + fullDataReturn --> loggingBuild["5. Build guardrail log payload"] + loggingBuild --> spendLogs["6a. Persist to spend logs / UI"] + loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"] +``` + +--- + +## Root Cause + +The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged. + +--- + +## Impact + +This issue required all of the following: + +1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`. +2. LiteLLM logged that guardrail response through the standard guardrail logging path. +3. An operator, admin, or telemetry consumer had access to the resulting logs or traces. + +When those conditions were met, sensitive values could become visible through: + +- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI. +- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans. +- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values. + +This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems. + +--- + +## Guidance For Users + +- Upgrade to LiteLLM 1.82.3+. +- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period. +- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems. +- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata. diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index f5b611a85a..042af2bfb4 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -902,6 +902,7 @@ router_settings: | OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry | OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing | OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console) +| OTEL_IGNORE_CONTEXT_PROPAGATION | When true, ignore parent span context propagation in OpenTelemetry callbacks | PAGERDUTY_API_KEY | API key for PagerDuty Alerting | PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service | PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ee3c344169..29d28b8c89 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -757,7 +757,7 @@ def _map_traffic_type_to_service_tier(traffic_type: Optional[str]) -> Optional[s """ if traffic_type is None: return None - service_tier = _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER.get(traffic_type.upper()) + service_tier = _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER.get(str(traffic_type).upper()) return service_tier diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 64e0b26a8e..da7e84a025 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -291,7 +291,7 @@ class DataDogLogger( dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=get_datadog_tags(), + ddtags=",".join(get_datadog_tags()), hostname=get_datadog_hostname(), message=safe_dumps(message_payload), service=get_datadog_service(), @@ -442,7 +442,7 @@ class DataDogLogger( verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload) dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=get_datadog_tags(standard_logging_object=standard_logging_object), + ddtags=",".join(get_datadog_tags(standard_logging_object=standard_logging_object)), hostname=get_datadog_hostname(), message=json_payload, service=get_datadog_service(), @@ -545,7 +545,7 @@ class DataDogLogger( _dd_message_str = safe_dumps(_payload_dict) _dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=get_datadog_tags(), + ddtags=",".join(get_datadog_tags()), hostname=get_datadog_hostname(), message=_dd_message_str, service=get_datadog_service(), @@ -587,7 +587,7 @@ class DataDogLogger( _dd_message_str = safe_dumps(_payload_dict) _dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=get_datadog_tags(), + ddtags=",".join(get_datadog_tags()), hostname=get_datadog_hostname(), message=_dd_message_str, service=get_datadog_service(), @@ -678,7 +678,7 @@ class DataDogLogger( dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=get_datadog_tags(), + ddtags=",".join(get_datadog_tags()), hostname=get_datadog_hostname(), message=json_payload, service=get_datadog_service(), diff --git a/litellm/integrations/datadog/datadog_handler.py b/litellm/integrations/datadog/datadog_handler.py index 0406f1e5d2..b6bb2b5703 100644 --- a/litellm/integrations/datadog/datadog_handler.py +++ b/litellm/integrations/datadog/datadog_handler.py @@ -38,8 +38,13 @@ def get_datadog_pod_name() -> str: def get_datadog_tags( standard_logging_object: Optional[StandardLoggingPayload] = None, -) -> str: - """Build Datadog tags string used by multiple integrations.""" +) -> List[str]: + """Build Datadog tags as a list of individual tag strings. + + Returns a list of "key:value" strings suitable for Datadog LLM Observability + (which expects tags as an array). For Datadog Logs API (ddtags), join with + comma: ",".join(get_datadog_tags(...)). + """ base_tags = { "env": get_datadog_env(), @@ -66,4 +71,4 @@ def get_datadog_tags( if team_tag: tags.append(f"team:{team_tag}") - return ",".join(tags) + return tags diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index de6cc02fa3..ec6c00961b 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -203,7 +203,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): type="span", attributes=DDSpanAttributes( ml_app=get_datadog_service(), - tags=[get_datadog_tags()], + tags=get_datadog_tags(), spans=self.log_queue, ), ), @@ -315,7 +315,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): duration=int((end_time - start_time).total_seconds() * 1e9), metrics=metrics, status="error" if error_info else "ok", - tags=[get_datadog_tags(standard_logging_object=standard_logging_payload)], + tags=get_datadog_tags(standard_logging_object=standard_logging_payload), ) apm_trace_id = self._get_apm_trace_id() diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6e991e6911..ca78e72c69 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1893,15 +1893,23 @@ class CustomStreamWrapper: "usage", getattr(complete_streaming_response, "usage"), ) - self.cache_streaming_response( - processed_chunk=complete_streaming_response.model_copy( + try: + _cache_copy = complete_streaming_response.model_copy( deep=True - ), + ) + _log_copy = complete_streaming_response.model_copy( + deep=True + ) + except RuntimeError: + _cache_copy = complete_streaming_response.model_copy() + _log_copy = complete_streaming_response.model_copy() + self.cache_streaming_response( + processed_chunk=_cache_copy, cache_hit=cache_hit, ) executor.submit( self.logging_obj.success_handler, - complete_streaming_response.model_copy(deep=True), + _log_copy, None, None, cache_hit, @@ -2113,11 +2121,15 @@ class CustomStreamWrapper: "usage", getattr(complete_streaming_response, "usage"), ) + try: + _copy = complete_streaming_response.model_copy( + deep=True + ) + except RuntimeError: + _copy = complete_streaming_response.model_copy() asyncio.create_task( self.async_cache_streaming_response( - processed_chunk=complete_streaming_response.model_copy( - deep=True - ), + processed_chunk=_copy, cache_hit=cache_hit, ) ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 181045809f..e7ff57f27e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4462,6 +4462,78 @@ "supports_vision": true, "supports_web_search": true }, + "azure/gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false + }, + "azure/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false + }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 3385e7feef..07309eb57f 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -208,26 +208,52 @@ async def exchange_token_with_server( client_id: str, client_secret: Optional[str], code_verifier: Optional[str], + refresh_token: Optional[str] = None, + scope: Optional[str] = None, ): - if grant_type != "authorization_code": + if grant_type not in ("authorization_code", "refresh_token"): raise HTTPException(status_code=400, detail="Unsupported grant_type") if mcp_server.token_url is None: raise HTTPException(status_code=400, detail="MCP server token url is not set") - proxy_base_url = get_request_base_url(request) - token_data = { - "grant_type": "authorization_code", - "client_id": mcp_server.client_id if mcp_server.client_id else client_id, - "client_secret": mcp_server.client_secret - if mcp_server.client_secret - else client_secret, - "code": code, - "redirect_uri": f"{proxy_base_url}/callback", - } + resolved_client_id = mcp_server.client_id if mcp_server.client_id else client_id + resolved_client_secret = ( + mcp_server.client_secret if mcp_server.client_secret else client_secret + ) - if code_verifier: - token_data["code_verifier"] = code_verifier + if grant_type == "refresh_token": + if not refresh_token: + raise HTTPException( + status_code=400, + detail="refresh_token is required for refresh_token grant", + ) + token_data: dict = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": resolved_client_id, + } + if resolved_client_secret is not None: + token_data["client_secret"] = resolved_client_secret + if scope: + token_data["scope"] = scope + else: + if not code: + raise HTTPException( + status_code=400, + detail="code is required for authorization_code grant", + ) + proxy_base_url = get_request_base_url(request) + token_data = { + "grant_type": "authorization_code", + "client_id": resolved_client_id, + "code": code, + "redirect_uri": f"{proxy_base_url}/callback", + } + if resolved_client_secret is not None: + token_data["client_secret"] = resolved_client_secret + if code_verifier: + token_data["code_verifier"] = code_verifier async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check) response = await async_client.post( @@ -375,6 +401,8 @@ async def token_endpoint( client_id: str = Form(...), client_secret: Optional[str] = Form(None), code_verifier: str = Form(None), + refresh_token: Optional[str] = Form(None), + scope: Optional[str] = Form(None), mcp_server_name: Optional[str] = None, ): """ @@ -408,6 +436,8 @@ async def token_endpoint( client_id=client_id, client_secret=client_secret, code_verifier=code_verifier, + refresh_token=refresh_token, + scope=scope, ) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index d31a13e8bc..6cf1f7ed6b 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -29,6 +29,7 @@ from litellm.constants import ( DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, ) +from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.proxy._types import ( RBAC_ROLES, @@ -407,18 +408,19 @@ async def common_checks( # noqa: PLR0915 # 2. If team can call model if _model and team_object: - if not await can_team_access_model( - model=_model, - team_object=team_object, - llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, - ): - raise ProxyException( - message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", - type=ProxyErrorTypes.team_model_access_denied, - param="model", - code=status.HTTP_401_UNAUTHORIZED, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_team_access_model"): + if not await can_team_access_model( + model=_model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=valid_token.team_model_aliases if valid_token else None, + ): + raise ProxyException( + message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", + type=ProxyErrorTypes.team_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) # Require trace id for agent keys when agent has require_trace_id_on_calls_by_agent if valid_token is not None and valid_token.agent_id: @@ -443,54 +445,60 @@ async def common_checks( # noqa: PLR0915 ## 2.1 If user can call model (if personal key) if _model and team_object is None and user_object is not None: - await can_user_call_model( - model=_model, - llm_router=llm_router, - user_object=user_object, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): + await can_user_call_model( + model=_model, + llm_router=llm_router, + user_object=user_object, + ) # 1.1 - 2.2 - 3.0.2 - 3.0.3: Project checks (blocked, model access, budget) - await _run_project_checks( - project_object=project_object, - _model=_model, - llm_router=llm_router, - skip_budget_checks=skip_budget_checks, - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.run_project_checks"): + await _run_project_checks( + project_object=project_object, + _model=_model, + llm_router=llm_router, + skip_budget_checks=skip_budget_checks, + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + ) # If this is a free model, skip all budget checks if not skip_budget_checks: # 3. If team is in budget - await _team_max_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.team_max_budget_check"): + await _team_max_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.0.5. If team is over soft budget (alert only, doesn't block) - await _team_soft_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.team_soft_budget_check"): + await _team_soft_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.1. If organization is in budget - await _organization_max_budget_check( - valid_token=valid_token, - team_object=team_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.organization_max_budget_check"): + await _organization_max_budget_check( + valid_token=valid_token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) - await _tag_max_budget_check( - request_body=request_body, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.tag_max_budget_check"): + await _tag_max_budget_check( + request_body=request_body, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 4. If user is in budget ## 4.1 check personal budget, if personal key @@ -508,14 +516,15 @@ async def common_checks( # noqa: PLR0915 ) ## 4.2 check team member budget, if team key - await _check_team_member_budget( - team_object=team_object, - user_object=user_object, - valid_token=valid_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.check_team_member_budget"): + await _check_team_member_budget( + team_object=team_object, + user_object=user_object, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # 5. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget if ( @@ -554,19 +563,21 @@ async def common_checks( # noqa: PLR0915 ) # 11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store - await vector_store_access_check( - request_body=request_body, - team_object=team_object, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.vector_store_access_check"): + await vector_store_access_check( + request_body=request_body, + team_object=team_object, + valid_token=valid_token, + ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) - await check_tools_allowlist( - request_body=request_body, - valid_token=valid_token, - team_object=team_object, - route=route, - ) + with tracer.trace("litellm.proxy.auth.common_checks.check_tools_allowlist"): + await check_tools_allowlist( + request_body=request_body, + valid_token=valid_token, + team_object=team_object, + route=route, + ) return True diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 044333ac13..30e59f77e6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -548,13 +548,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 custom_auth_api_key: bool = False try: - # get the request body - - await pre_db_read_auth_checks( - request_data=request_data, - request=request, - route=route, - ) + with tracer.trace("litellm.proxy.auth.pre_db_read_auth_checks"): + await pre_db_read_auth_checks( + request_data=request_data, + request=request, + route=route, + ) pass_through_endpoints: Optional[List[dict]] = general_settings.get( "pass_through_endpoints", None ) @@ -588,9 +587,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ### USER-DEFINED AUTH FUNCTION ### if enterprise_custom_auth is not None: - response = await enterprise_custom_auth( - request=request, api_key=api_key, user_custom_auth=user_custom_auth - ) + with tracer.trace("litellm.proxy.auth.enterprise_custom_auth"): + response = await enterprise_custom_auth( + request=request, api_key=api_key, user_custom_auth=user_custom_auth + ) if response is not None and isinstance(response, UserAPIKeyAuth): validated = UserAPIKeyAuth.model_validate(response) validated = await _run_post_custom_auth_checks( @@ -706,18 +706,19 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # Fall through to virtual key checks if do_standard_jwt_auth: - result = await JWTAuthManager.auth_builder( - request_data=request_data, - general_settings=general_settings, - api_key=api_key, - jwt_handler=jwt_handler, - route=route, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - parent_otel_span=parent_otel_span, - request_headers=_safe_get_request_headers(request), - ) + with tracer.trace("litellm.proxy.auth.jwt_auth_builder"): + result = await JWTAuthManager.auth_builder( + request_data=request_data, + general_settings=general_settings, + api_key=api_key, + jwt_handler=jwt_handler, + route=route, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + parent_otel_span=parent_otel_span, + request_headers=_safe_get_request_headers(request), + ) is_proxy_admin = result["is_proxy_admin"] team_id = result["team_id"] @@ -909,15 +910,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 try: end_user_params["end_user_id"] = end_user_id - # get end-user object - _end_user_object = await get_end_user_object( - end_user_id=end_user_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - route=route, - ) + with tracer.trace("litellm.proxy.auth.get_end_user_object"): + _end_user_object = await get_end_user_object( + end_user_id=end_user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + route=route, + ) if _end_user_object is not None: end_user_params[ "allowed_model_region" @@ -960,14 +961,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 if valid_token is None: ## Check CACHE try: - valid_token = await get_key_object( - hashed_token=hash_token(api_key), - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - check_cache_only=True, - ) + with tracer.trace("litellm.proxy.auth.get_key_object_check_cache"): + valid_token = await get_key_object( + hashed_token=hash_token(api_key), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + check_cache_only=True, + ) except Exception: verbose_logger.debug("api key not found in cache.") valid_token = None @@ -1139,13 +1141,14 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 api_key = hash_token(token=api_key) try: - valid_token = await get_key_object( - hashed_token=api_key, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.get_key_object_from_db"): + valid_token = await get_key_object( + hashed_token=api_key, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) except ProxyException as e: if e.code == 401 or e.code == "401": e.message = "Authentication Error, Invalid proxy server token passed. Received API Key = {}, Key Hash (Token) ={}. Unable to find token in cache or `LiteLLM_VerificationTokenTable`".format( @@ -1233,14 +1236,15 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 # Check 2. If user_id for this token is in budget - done in common_checks() if valid_token.user_id is not None: try: - user_obj = await get_user_object( - user_id=valid_token.user_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - user_id_upsert=False, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.get_user_object"): + user_obj = await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) except Exception as e: verbose_logger.debug( "litellm.proxy.auth.user_api_key_auth.py::user_api_key_auth() - Unable to get user from db/cache. Setting user_obj to None. Exception received - {}".format( @@ -1329,71 +1333,73 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) if not skip_budget_checks: - # Check 4. Token Spend is under budget - if RouteChecks.is_llm_api_route(route=route): - await _virtual_key_max_budget_check( + with tracer.trace("litellm.proxy.auth.budget_checks"): + # Check 4. Token Spend is under budget + if RouteChecks.is_llm_api_route(route=route): + await _virtual_key_max_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, + ) + + # Check 5. Max Budget Alert Check + await _virtual_key_max_budget_alert_check( valid_token=valid_token, proxy_logging_obj=proxy_logging_obj, user_obj=user_obj, ) - # Check 5. Max Budget Alert Check - await _virtual_key_max_budget_alert_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 6. Soft Budget Check - await _virtual_key_soft_budget_check( - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - user_obj=user_obj, - ) - - # Check 5. Token Model Spend is under Model budget - max_budget_per_model = valid_token.model_max_budget - current_model = request_data.get("model", None) - - if ( - max_budget_per_model is not None - and isinstance(max_budget_per_model, dict) - and len(max_budget_per_model) > 0 - and prisma_client is not None - and current_model is not None - and valid_token.token is not None - ): - ## GET THE SPEND FOR THIS MODEL - await model_max_budget_limiter.is_key_within_model_budget( - user_api_key_dict=valid_token, - model=current_model, + # Check 6. Soft Budget Check + await _virtual_key_soft_budget_check( + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + user_obj=user_obj, ) - # Check 5b. End-user model max budget - end_user_mmb = valid_token.end_user_model_max_budget - if ( - end_user_mmb is not None - and isinstance(end_user_mmb, dict) - and len(end_user_mmb) > 0 - and current_model is not None - and valid_token.end_user_id is not None - ): - await model_max_budget_limiter.is_end_user_within_model_budget( - end_user_id=valid_token.end_user_id, - end_user_model_max_budget=end_user_mmb, - model=current_model, - ) + # Check 5. Token Model Spend is under Model budget + max_budget_per_model = valid_token.model_max_budget + current_model = request_data.get("model", None) + + if ( + max_budget_per_model is not None + and isinstance(max_budget_per_model, dict) + and len(max_budget_per_model) > 0 + and prisma_client is not None + and current_model is not None + and valid_token.token is not None + ): + ## GET THE SPEND FOR THIS MODEL + await model_max_budget_limiter.is_key_within_model_budget( + user_api_key_dict=valid_token, + model=current_model, + ) + + # Check 5b. End-user model max budget + end_user_mmb = valid_token.end_user_model_max_budget + if ( + end_user_mmb is not None + and isinstance(end_user_mmb, dict) + and len(end_user_mmb) > 0 + and current_model is not None + and valid_token.end_user_id is not None + ): + await model_max_budget_limiter.is_end_user_within_model_budget( + end_user_id=valid_token.end_user_id, + end_user_model_max_budget=end_user_mmb, + model=current_model, + ) # Check 6: Additional Common Checks across jwt + key auth if valid_token.team_id is not None: try: - _team_obj = await get_team_object( - team_id=valid_token.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.get_team_object"): + _team_obj = await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) except HTTPException: _team_obj = LiteLLM_TeamTableCachedObj( team_id=valid_token.team_id, @@ -1431,11 +1437,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 litellm.max_budget > 0 and prisma_client is not None ): # user set proxy max budget cache_key = "{}:spend".format(litellm_proxy_admin_name) - global_proxy_spend = await _fetch_global_spend_with_event_coordination( - cache_key=cache_key, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, - ) + with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): + global_proxy_spend = await _fetch_global_spend_with_event_coordination( + cache_key=cache_key, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + ) if global_proxy_spend is not None: call_info = CallInfo( @@ -1452,21 +1459,22 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_info=call_info, ) ) - _ = await common_checks( - request=request, - request_body=request_data, - team_object=_team_obj, - user_object=user_obj, - end_user_object=_end_user_object, - general_settings=general_settings, - global_proxy_spend=global_proxy_spend, - route=route, - llm_router=llm_router, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - skip_budget_checks=skip_budget_checks, - project_object=_project_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks"): + _ = await common_checks( + request=request, + request_body=request_data, + team_object=_team_obj, + user_object=user_obj, + end_user_object=_end_user_object, + general_settings=general_settings, + global_proxy_spend=global_proxy_spend, + route=route, + llm_router=llm_router, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + skip_budget_checks=skip_budget_checks, + project_object=_project_obj, + ) # Token passed all checks if valid_token is None: raise HTTPException(401, detail="Invalid API key") diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 72765aab7d..e5a31c3671 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1260,7 +1260,9 @@ class ProxyBaseLLMRequestProcessing: custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, call_id=( - _litellm_logging_obj.litellm_call_id if _litellm_logging_obj else None + _litellm_logging_obj.litellm_call_id + if _litellm_logging_obj + else self.data.get("litellm_call_id") ), model_id=model_id, version=version, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 3e5b729cea..f29a721ede 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -1399,6 +1399,8 @@ if MCP_AVAILABLE: client_id: Optional[str] = Form(None), client_secret: Optional[str] = Form(None), code_verifier: Optional[str] = Form(None), + refresh_token: Optional[str] = Form(None), + scope: Optional[str] = Form(None), ): mcp_server = _get_cached_temporary_mcp_server_or_404(server_id) resolved_client_id = mcp_server.client_id or client_id or "" @@ -1422,6 +1424,8 @@ if MCP_AVAILABLE: client_id=resolved_client_id, client_secret=client_secret, code_verifier=code_verifier, + refresh_token=refresh_token, + scope=scope, ) @router.post( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 181045809f..879dd42be4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4462,6 +4462,78 @@ "supports_vision": true, "supports_web_search": true }, + "azure/gpt-5.4-mini": { + "cache_read_input_token_cost": 7.5e-08, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false + }, + "azure/gpt-5.4-nano": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "azure", + "max_input_tokens": 1050000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1.25e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/batch", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": false, + "supports_xhigh_reasoning_effort": false + }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, "cache_read_input_token_cost": 1.25e-06, @@ -37032,5 +37104,157 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true + }, + "volcengine/doubao-seed-2-0-pro-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 4.6e-07, + "output_cost_per_token": 2.3e-06, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 3.5e-06, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 7e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-lite-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 8.7e-08, + "output_cost_per_token": 5.2e-07, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 1.3e-07, + "output_cost_per_token": 7.8e-07, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 2.6e-07, + "output_cost_per_token": 1.6e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-mini-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 2.9e-08, + "output_cost_per_token": 2.9e-07, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 5.8e-08, + "output_cost_per_token": 5.8e-07, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 1.2e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] + }, + "volcengine/doubao-seed-2-0-code-preview-260215": { + "litellm_provider": "volcengine", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "source": "https://www.volcengine.com/docs/82379/1330310", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": false, + "supports_vision": true, + "tiered_pricing": [ + { + "input_cost_per_token": 4.6e-07, + "output_cost_per_token": 2.3e-06, + "range": [ + 0, + 32000.0 + ] + }, + { + "input_cost_per_token": 7e-07, + "output_cost_per_token": 3.5e-06, + "range": [ + 32000.0, + 128000.0 + ] + }, + { + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 7e-06, + "range": [ + 128000.0, + 256000.0 + ] + } + ] } } diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index fc4b3ff3cf..4cfd4a6cc9 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -593,7 +593,7 @@ def test_datadog_static_methods(): # Test tags format with default values assert ( "env:unknown,service:litellm-server,version:unknown,HOSTNAME:" - in get_datadog_tags() + in ",".join(get_datadog_tags()) ) # Test with custom environment variables @@ -631,7 +631,7 @@ def test_datadog_static_methods(): # Test tags format with custom values expected_custom_tags = "env:production,service:custom-service,version:1.0.0,HOSTNAME:test-host,POD_NAME:pod-123" print("DataDogLogger._get_datadog_tags()", get_datadog_tags()) - assert get_datadog_tags() == expected_custom_tags + assert ",".join(get_datadog_tags()) == expected_custom_tags @pytest.mark.asyncio @@ -672,11 +672,11 @@ def test_get_datadog_tags(): """Test the _get_datadog_tags static method with various inputs""" # Test with no standard_logging_object and default env vars base_tags = get_datadog_tags() - assert "env:" in base_tags - assert "service:" in base_tags - assert "version:" in base_tags - assert "POD_NAME:" in base_tags - assert "HOSTNAME:" in base_tags + assert any("env:" in t for t in base_tags) + assert any("service:" in t for t in base_tags) + assert any("version:" in t for t in base_tags) + assert any("POD_NAME:" in t for t in base_tags) + assert any("HOSTNAME:" in t for t in base_tags) # Test with custom env vars test_env = { @@ -705,12 +705,12 @@ def test_get_datadog_tags(): # Test with empty request_tags standard_logging_obj["request_tags"] = [] tags_empty_request = get_datadog_tags(standard_logging_obj) - assert "request_tag:" not in tags_empty_request + assert not any(t.startswith("request_tag:") for t in tags_empty_request) # Test with None request_tags standard_logging_obj["request_tags"] = None tags_none_request = get_datadog_tags(standard_logging_obj) - assert "request_tag:" not in tags_none_request + assert not any(t.startswith("request_tag:") for t in tags_none_request) @pytest.mark.asyncio diff --git a/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py b/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py index 3f1d2be413..cc9eae7a37 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_tags_regression.py @@ -44,7 +44,7 @@ class TestDatadogTagsRegression: assert "env:test-env" in tags_legacy assert "service:test-service" in tags_legacy # Verify NO team tag (should not invent one) - assert "team:" not in tags_legacy + assert not any(t.startswith("team:") for t in tags_legacy) # Case 2: New feature (team info provided) payload_with_team = StandardLoggingPayload( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 700ba86b10..954f2703e3 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -1666,3 +1666,141 @@ async def test_oauth_authorize_prefers_request_scope_over_server_config(): redirect_url = response.headers["location"] assert "scope=custom_scope1+custom_scope2" in redirect_url or "scope=custom_scope1%20custom_scope2" in redirect_url assert "default_scope" not in redirect_url + + +@pytest.mark.asyncio +async def test_token_endpoint_refresh_token_grant(): + """Test that token endpoint supports refresh_token grant type.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + token_endpoint, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="google_mcp", + name="google_mcp", + server_name="google_mcp", + alias="google_mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_secret", + authorization_url="https://accounts.google.com/o/oauth2/v2/auth", + token_url="https://oauth2.googleapis.com/token", + scopes=["openid", "email"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://proxy.litellm.example/" + mock_request.headers = {} + + # Mock httpx client response with new tokens + mock_response = MagicMock() + mock_response.json.return_value = { + "access_token": "new_access_token", + "token_type": "Bearer", + "expires_in": 3599, + "refresh_token": "new_refresh_token", + } + mock_response.raise_for_status = MagicMock() + + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=mock_response) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client" + ) as mock_get_client: + mock_get_client.return_value = mock_async_client + + response = await token_endpoint( + request=mock_request, + grant_type="refresh_token", + code=None, + redirect_uri=None, + client_id="test_client_id", + mcp_server_name="google_mcp", + client_secret="test_secret", + refresh_token="rt-test", + scope="openid email", + ) + + # Verify the POST was called with refresh_token grant data + mock_async_client.post.assert_called_once() + call_args = mock_async_client.post.call_args + + assert call_args[1]["data"]["grant_type"] == "refresh_token" + assert call_args[1]["data"]["refresh_token"] == "rt-test" + assert call_args[1]["data"]["client_id"] == "test_client_id" + assert call_args[1]["data"]["client_secret"] == "test_secret" + assert call_args[1]["data"]["scope"] == "openid email" + + # Verify response contains the new tokens + import json + + token_data = json.loads(response.body) + assert token_data["access_token"] == "new_access_token" + assert token_data["refresh_token"] == "new_refresh_token" + + +@pytest.mark.asyncio +async def test_token_endpoint_authorization_code_missing_code(): + """Test that authorization_code grant rejects missing code param.""" + try: + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + + server = MCPServer( + server_id="test_server", + name="test_server", + server_name="test_server", + alias="test_server", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="cid", + token_url="https://example.com/token", + ) + global_mcp_server_manager.registry[server.server_id] = server + + mock_request = MagicMock() + mock_request.base_url = "https://proxy.example/" + mock_request.headers = {} + + with pytest.raises(HTTPException) as exc_info: + await exchange_token_with_server( + request=mock_request, + mcp_server=server, + grant_type="authorization_code", + code=None, + redirect_uri="https://example.com/cb", + client_id="cid", + client_secret=None, + code_verifier=None, + ) + assert exc_info.value.status_code == 400 + assert "code is required" in str(exc_info.value.detail) diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index eeaeb49832..77ac3a040a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1519,6 +1519,8 @@ class TestTemporaryMCPSessionEndpoints: client_id="client", client_secret="secret", code_verifier="verifier", + refresh_token=None, + scope=None, ) assert result is exchange_response @@ -1532,6 +1534,56 @@ class TestTemporaryMCPSessionEndpoints: client_id="client", client_secret="secret", code_verifier="verifier", + refresh_token=None, + scope=None, + ) + + @pytest.mark.asyncio + async def test_mcp_token_proxies_refresh_token_grant(self): + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + mcp_token, + ) + + request = MagicMock() + server = generate_mock_mcp_server_config_record(server_id="server-1") + exchange_response = {"access_token": "new-token", "refresh_token": "new-rt"} + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._get_cached_temporary_mcp_server_or_404", + return_value=server, + ) as get_server, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.exchange_token_with_server", + AsyncMock(return_value=exchange_response), + ) as exchange_mock, + ): + result = await mcp_token( + request=request, + server_id="server-1", + grant_type="refresh_token", + code=None, + redirect_uri=None, + client_id="client", + client_secret="secret", + code_verifier=None, + refresh_token="rt-123", + scope=None, + ) + + assert result is exchange_response + get_server.assert_called_once_with("server-1") + exchange_mock.assert_awaited_once_with( + request=request, + mcp_server=server, + grant_type="refresh_token", + code=None, + redirect_uri=None, + client_id="client", + client_secret="secret", + code_verifier=None, + refresh_token="rt-123", + scope=None, ) @pytest.mark.asyncio From c2b8ba8b1ba0832d34ed7432f3eb870e749e3c54 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 12:28:56 -0700 Subject: [PATCH 47/57] [Fix] Resolve mypy errors in key_management_endpoints.py Add None guard for prisma_client before calling update_data, and add "unblocked" to AUDIT_ACTIONS literal type. Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/_types.py | 2 +- litellm/proxy/management_endpoints/key_management_endpoints.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9e86680e35..f5568757a2 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2955,7 +2955,7 @@ class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): endTime: Union[str, datetime, None] -AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "rotated"] +AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated"] class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 7be1a5a5e0..5e573d0bbe 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2150,6 +2150,8 @@ async def update_key_fn( ) _data = {**non_default_values, "token": key} + if prisma_client is None: + raise Exception("Not connected to DB!") response = await prisma_client.update_data(token=key, data=_data) # Delete - key from cache, since it's been updated! From 004d8d01f6840ecf0b542b09293cb8c59c665787 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 12:39:39 -0700 Subject: [PATCH 48/57] chore: apply black formatting to fix lint CI --- litellm/integrations/datadog/datadog.py | 4 +++- litellm/litellm_core_utils/streaming_handler.py | 12 +++--------- litellm/proxy/auth/auth_checks.py | 8 ++++++-- litellm/proxy/auth/user_api_key_auth.py | 10 ++++++---- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index da7e84a025..4de3644b58 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -442,7 +442,9 @@ class DataDogLogger( verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload) dd_payload = DatadogPayload( ddsource=get_datadog_source(), - ddtags=",".join(get_datadog_tags(standard_logging_object=standard_logging_object)), + ddtags=",".join( + get_datadog_tags(standard_logging_object=standard_logging_object) + ), hostname=get_datadog_hostname(), message=json_payload, service=get_datadog_service(), diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index ca78e72c69..370b53244b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1894,12 +1894,8 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) try: - _cache_copy = complete_streaming_response.model_copy( - deep=True - ) - _log_copy = complete_streaming_response.model_copy( - deep=True - ) + _cache_copy = complete_streaming_response.model_copy(deep=True) + _log_copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: _cache_copy = complete_streaming_response.model_copy() _log_copy = complete_streaming_response.model_copy() @@ -2122,9 +2118,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) try: - _copy = complete_streaming_response.model_copy( - deep=True - ) + _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: _copy = complete_streaming_response.model_copy() asyncio.create_task( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 6cf1f7ed6b..1aa14fff57 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -413,7 +413,9 @@ async def common_checks( # noqa: PLR0915 model=_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, + team_model_aliases=valid_token.team_model_aliases + if valid_token + else None, ): raise ProxyException( message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", @@ -482,7 +484,9 @@ async def common_checks( # noqa: PLR0915 ) # 3.1. If organization is in budget - with tracer.trace("litellm.proxy.auth.common_checks.organization_max_budget_check"): + with tracer.trace( + "litellm.proxy.auth.common_checks.organization_max_budget_check" + ): await _organization_max_budget_check( valid_token=valid_token, team_object=team_object, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 30e59f77e6..eb6a5bdb99 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1438,10 +1438,12 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ): # user set proxy max budget cache_key = "{}:spend".format(litellm_proxy_admin_name) with tracer.trace("litellm.proxy.auth.get_global_proxy_spend"): - global_proxy_spend = await _fetch_global_spend_with_event_coordination( - cache_key=cache_key, - user_api_key_cache=user_api_key_cache, - prisma_client=prisma_client, + global_proxy_spend = ( + await _fetch_global_spend_with_event_coordination( + cache_key=cache_key, + user_api_key_cache=user_api_key_cache, + prisma_client=prisma_client, + ) ) if global_proxy_spend is not None: From cf6369770310844fc5855b8a47820f7f1fabf0bc Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 12:41:11 -0700 Subject: [PATCH 49/57] [Fix] Update tests for _get_and_validate_existing_key refactor Tests were outdated after _get_and_validate_existing_key was refactored to use prisma_client.db.litellm_verificationtoken.find_unique() instead of prisma_client.get_data(), and to raise ProxyException instead of HTTPException. Also fix bulk_update_keys error handler to extract ProxyException.message (str(ProxyException) returns empty string). Co-Authored-By: Claude Opus 4.6 --- .../key_management_endpoints.py | 2 + .../test_key_management_endpoints.py | 233 ++++++++++-------- 2 files changed, 128 insertions(+), 107 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5e573d0bbe..a550af31e7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,6 +2324,8 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) + elif isinstance(e, ProxyException): + error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 48a1f7936c..37551fe387 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5108,6 +5108,8 @@ async def test_get_and_validate_existing_key(): """ from fastapi import HTTPException + from litellm.proxy.utils import ProxyException + # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5116,31 +5118,37 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.get_data = AsyncMock(return_value=mock_key) - - result = await _get_and_validate_existing_key( - token="test-key-123", - prisma_client=mock_prisma_client, - ) - - assert result == mock_key - mock_prisma_client.get_data.assert_called_once_with( - token="test-key-123", - table_name="key", - query_type="find_unique", - ) - - # Test Case 2: Key not found raises HTTPException - mock_prisma_client.get_data = AsyncMock(return_value=None) - - with pytest.raises(HTTPException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=mock_key) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", + ): + result = await _get_and_validate_existing_key( + token="test-key-123", prisma_client=mock_prisma_client, ) - - assert exc_info.value.status_code == 404 - assert "Key not found" in str(exc_info.value.detail) + + assert result == mock_key + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": "hashed-test-key-123"} + ) + + # Test Case 2: Key not found raises ProxyException + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-non-existent-key", + ): + with pytest.raises(ProxyException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", + prisma_client=mock_prisma_client, + ) + + assert str(exc_info.value.code) == "404" + assert "Key not found" in str(exc_info.value.message) # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: @@ -5189,75 +5197,80 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.get_data = AsyncMock(return_value=existing_key) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing_key) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock KeyManagementEventHooks + + # Mock _hash_token_if_needed with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.get_data.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Mock KeyManagementEventHooks + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5447,15 +5460,17 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) + # get_data is used by the error handler to fetch key info for failed updates + mock_prisma_client.get_data = AsyncMock(return_value=None) mock_updated_key_1_obj = MagicMock() mock_updated_key_1_obj.model_dump.return_value = updated_key_1_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5467,13 +5482,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5484,46 +5499,50 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=lambda token: f"hashed-{token}", ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From e86ca7f34d41d764d885fc64ce86dc97508035f8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 14:32:14 -0700 Subject: [PATCH 50/57] Revert "[Fix] Update tests for _get_and_validate_existing_key refactor" This reverts commit cf6369770310844fc5855b8a47820f7f1fabf0bc. --- .../key_management_endpoints.py | 2 - .../test_key_management_endpoints.py | 233 ++++++++---------- 2 files changed, 107 insertions(+), 128 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a550af31e7..5e573d0bbe 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,8 +2324,6 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) - elif isinstance(e, ProxyException): - error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 37551fe387..48a1f7936c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5108,8 +5108,6 @@ async def test_get_and_validate_existing_key(): """ from fastapi import HTTPException - from litellm.proxy.utils import ProxyException - # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5118,37 +5116,31 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=mock_key) - - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-test-key-123", - ): - result = await _get_and_validate_existing_key( - token="test-key-123", + mock_prisma_client.get_data = AsyncMock(return_value=mock_key) + + result = await _get_and_validate_existing_key( + token="test-key-123", + prisma_client=mock_prisma_client, + ) + + assert result == mock_key + mock_prisma_client.get_data.assert_called_once_with( + token="test-key-123", + table_name="key", + query_type="find_unique", + ) + + # Test Case 2: Key not found raises HTTPException + mock_prisma_client.get_data = AsyncMock(return_value=None) + + with pytest.raises(HTTPException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", prisma_client=mock_prisma_client, ) - - assert result == mock_key - mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( - where={"token": "hashed-test-key-123"} - ) - - # Test Case 2: Key not found raises ProxyException - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-non-existent-key", - ): - with pytest.raises(ProxyException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", - prisma_client=mock_prisma_client, - ) - - assert str(exc_info.value.code) == "404" - assert "Key not found" in str(exc_info.value.message) + + assert exc_info.value.status_code == 404 + assert "Key not found" in str(exc_info.value.detail) # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: @@ -5197,80 +5189,75 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing_key) + mock_prisma_client.get_data = AsyncMock(return_value=existing_key) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock _hash_token_if_needed + + # Mock KeyManagementEventHooks with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - return_value="hashed-test-key-123", + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): - # Mock KeyManagementEventHooks - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" - ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.get_data.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5460,17 +5447,15 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + mock_prisma_client.get_data = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) - # get_data is used by the error handler to fetch key info for failed updates - mock_prisma_client.get_data = AsyncMock(return_value=None) mock_updated_key_1_obj = MagicMock() mock_updated_key_1_obj.model_dump.return_value = updated_key_1_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5482,13 +5467,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5499,50 +5484,46 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", - side_effect=lambda token: f"hashed-{token}", + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" ): - with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" - ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From d118bf48188fb72ebd23f625df5a72189a052114 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 14:36:02 -0700 Subject: [PATCH 51/57] chore: add poetry check --lock to lint CI to prevent stale lockfile merges --- .github/workflows/test-linting.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index fc0f84a20d..4cedb8b5ba 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -28,9 +28,12 @@ jobs: find . -type d -name "__pycache__" -exec rm -rf {} + || true find . -name "*.pyc" -delete || true + - name: Check poetry.lock is up to date + run: | + poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1) + - name: Install dependencies run: | - poetry lock poetry install --with dev - name: Check Black formatting From 05620c87e38b7cc03a9483db7f334c5b8aa2eb0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 15:34:44 -0700 Subject: [PATCH 52/57] [Fix] Update bulk key update tests for find_unique refactor Tests were outdated after _get_and_validate_existing_key was refactored to use prisma_client.db.litellm_verificationtoken.find_unique() and ProxyException. Also add ProxyException handling in bulk_update_keys error extractor so error messages aren't empty. Co-Authored-By: Claude Opus 4.6 --- .../key_management_endpoints.py | 2 + .../test_key_management_endpoints.py | 331 ++++++++++-------- 2 files changed, 182 insertions(+), 151 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5e573d0bbe..a550af31e7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2324,6 +2324,8 @@ async def bulk_update_keys( error_message = error_detail.get("error", str(e)) else: error_message = str(error_detail) + elif isinstance(e, ProxyException): + error_message = e.message else: error_message = str(e) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 48a1f7936c..12ec79d3e0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5100,14 +5100,16 @@ async def test_validate_max_budget(): async def test_get_and_validate_existing_key(): """ Test _get_and_validate_existing_key helper function. - + Tests: 1. Successfully retrieve existing key - 2. Key not found raises HTTPException + 2. Key not found raises ProxyException 3. Database not connected raises HTTPException """ from fastapi import HTTPException + from litellm.proxy._types import ProxyException + # Test Case 1: Successfully retrieve existing key mock_prisma_client = AsyncMock() mock_key = LiteLLM_VerificationToken( @@ -5116,39 +5118,49 @@ async def test_get_and_validate_existing_key(): models=["gpt-4"], team_id=None, ) - mock_prisma_client.get_data = AsyncMock(return_value=mock_key) - - result = await _get_and_validate_existing_key( - token="test-key-123", - prisma_client=mock_prisma_client, + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=mock_key ) - - assert result == mock_key - mock_prisma_client.get_data.assert_called_once_with( - token="test-key-123", - table_name="key", - query_type="find_unique", - ) - - # Test Case 2: Key not found raises HTTPException - mock_prisma_client.get_data = AsyncMock(return_value=None) - - with pytest.raises(HTTPException) as exc_info: - await _get_and_validate_existing_key( - token="non-existent-key", + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", + ): + result = await _get_and_validate_existing_key( + token="test-key-123", prisma_client=mock_prisma_client, ) - - assert exc_info.value.status_code == 404 - assert "Key not found" in str(exc_info.value.detail) - + + assert result == mock_key + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once_with( + where={"token": "hashed-test-key-123"} + ) + + # Test Case 2: Key not found raises ProxyException + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-non-existent-key", + ): + with pytest.raises(ProxyException) as exc_info: + await _get_and_validate_existing_key( + token="non-existent-key", + prisma_client=mock_prisma_client, + ) + + assert str(exc_info.value.code) == "404" + assert "Key not found" in exc_info.value.message + # Test Case 3: Database not connected raises HTTPException with pytest.raises(HTTPException) as exc_info: await _get_and_validate_existing_key( token="test-key-123", prisma_client=None, ) - + assert exc_info.value.status_code == 500 assert "Database not connected" in str(exc_info.value.detail) @@ -5189,75 +5201,82 @@ async def test_process_single_key_update(): "tags": ["production"], } - mock_prisma_client.get_data = AsyncMock(return_value=existing_key) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=existing_key + ) mock_updated_key_obj = MagicMock() mock_updated_key_obj.model_dump.return_value = updated_key_data mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_obj} ) - + # Mock prepare_key_update_data with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + # Mock TeamMemberPermissionChecks with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ) as mock_permission_check: mock_permission_check.return_value = None - + # Mock _delete_cache_key_object with patch( "litellm.proxy.management_endpoints.key_management_endpoints._delete_cache_key_object" ) as mock_delete_cache: mock_delete_cache.return_value = None - + # Mock hash_token (imported from litellm.proxy._types) with patch( "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-test-key-123" - - # Mock KeyManagementEventHooks + + # Mock _hash_token_if_needed with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + return_value="hashed-test-key-123", ): - # Create update request - key_update_item = BulkUpdateKeyRequestItem( - key="test-key-123", - max_budget=100.0, - tags=["production"], - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call the function - result = await _process_single_key_update( - key_update_item=key_update_item, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - prisma_client=mock_prisma_client, - user_api_key_cache=mock_user_api_key_cache, - proxy_logging_obj=mock_proxy_logging_obj, - llm_router=mock_llm_router, - ) - - # Verify results - assert result is not None - assert "token" not in result # Token should be removed - assert result.get("max_budget") == 100.0 - assert result.get("tags") == ["production"] - - # Verify mocks were called - mock_prisma_client.get_data.assert_called_once() - mock_prisma_client.update_data.assert_called_once() - mock_delete_cache.assert_called_once() + # Mock KeyManagementEventHooks + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create update request + key_update_item = BulkUpdateKeyRequestItem( + key="test-key-123", + max_budget=100.0, + tags=["production"], + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call the function + result = await _process_single_key_update( + key_update_item=key_update_item, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + llm_router=mock_llm_router, + ) + + # Verify results + assert result is not None + assert "token" not in result # Token should be removed + assert result.get("max_budget") == 100.0 + assert result.get("tags") == ["production"] + + # Verify mocks were called + mock_prisma_client.db.litellm_verificationtoken.find_unique.assert_called_once() + mock_prisma_client.update_data.assert_called_once() + mock_delete_cache.assert_called_once() @pytest.mark.asyncio @@ -5319,7 +5338,7 @@ async def test_bulk_update_keys_success(monkeypatch): "tags": ["staging"], } - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, existing_key_2] ) mock_updated_key_1_obj = MagicMock() @@ -5332,7 +5351,7 @@ async def test_bulk_update_keys_success(monkeypatch): {"data": mock_updated_key_2_obj}, ] ) - + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5344,7 +5363,7 @@ async def test_bulk_update_keys_success(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" @@ -5353,7 +5372,7 @@ async def test_bulk_update_keys_success(monkeypatch): {"max_budget": 100.0, "tags": ["production"]}, {"max_budget": 200.0, "tags": ["staging"]}, ] - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5364,45 +5383,49 @@ async def test_bulk_update_keys_success(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.side_effect = ["hashed-key-1", "hashed-key-2"] - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=["hashed-key-1", "hashed-key-2"], ): - # Create request - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="test-key-2", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 2 - assert len(response.failed_updates) == 0 - assert response.successful_updates[0].key == "test-key-1" - assert response.successful_updates[1].key == "test-key-2" + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="test-key-2", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 2 + assert len(response.failed_updates) == 0 + assert response.successful_updates[0].key == "test-key-1" + assert response.successful_updates[1].key == "test-key-2" @pytest.mark.asyncio @@ -5447,7 +5470,7 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): } # First key exists, second key doesn't exist - mock_prisma_client.get_data = AsyncMock( + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( side_effect=[existing_key_1, None] # Second key not found ) mock_updated_key_1_obj = MagicMock() @@ -5455,7 +5478,9 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): mock_prisma_client.update_data = AsyncMock( return_value={"data": mock_updated_key_1_obj} ) - + # Mock get_data for the error handler path (used to fetch key_info on failure) + mock_prisma_client.get_data = AsyncMock(return_value=None) + # Patch dependencies monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", mock_prisma_client @@ -5467,13 +5492,13 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging_obj ) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mock_llm_router) - + # Mock helper functions with patch( "litellm.proxy.management_endpoints.key_management_endpoints.prepare_key_update_data" ) as mock_prepare: mock_prepare.return_value = {"max_budget": 100.0, "tags": ["production"]} - + with patch( "litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint" ): @@ -5484,46 +5509,50 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): "litellm.proxy._types.hash_token" ) as mock_hash: mock_hash.return_value = "hashed-key-1" - + with patch( - "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + "litellm.proxy.management_endpoints.key_management_endpoints._hash_token_if_needed", + side_effect=["hashed-key-1", "hashed-non-existent-key"], ): - # Create request with one valid and one invalid key - request_data = BulkUpdateKeyRequest( - keys=[ - BulkUpdateKeyRequestItem( - key="test-key-1", - max_budget=100.0, - tags=["production"], - ), - BulkUpdateKeyRequestItem( - key="non-existent-key", - max_budget=200.0, - tags=["staging"], - ), - ] - ) - - user_api_key_dict = UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN, - api_key="sk-admin", - user_id="admin-user", - ) - - # Call endpoint - response = await bulk_update_keys( - data=request_data, - user_api_key_dict=user_api_key_dict, - litellm_changed_by=None, - ) - - # Verify response - assert response.total_requested == 2 - assert len(response.successful_updates) == 1 - assert len(response.failed_updates) == 1 - assert response.successful_updates[0].key == "test-key-1" - assert response.failed_updates[0].key == "non-existent-key" - assert "Key not found" in response.failed_updates[0].failed_reason + with patch( + "litellm.proxy.management_endpoints.key_management_endpoints.KeyManagementEventHooks.async_key_updated_hook" + ): + # Create request with one valid and one invalid key + request_data = BulkUpdateKeyRequest( + keys=[ + BulkUpdateKeyRequestItem( + key="test-key-1", + max_budget=100.0, + tags=["production"], + ), + BulkUpdateKeyRequestItem( + key="non-existent-key", + max_budget=200.0, + tags=["staging"], + ), + ] + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-admin", + user_id="admin-user", + ) + + # Call endpoint + response = await bulk_update_keys( + data=request_data, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=None, + ) + + # Verify response + assert response.total_requested == 2 + assert len(response.successful_updates) == 1 + assert len(response.failed_updates) == 1 + assert response.successful_updates[0].key == "test-key-1" + assert response.failed_updates[0].key == "non-existent-key" + assert "Key not found" in response.failed_updates[0].failed_reason @pytest.mark.parametrize( From f60e3cfd34f6a4523dbd4ba0d42c97343e0195f1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 16:29:51 -0700 Subject: [PATCH 53/57] remove returning key in error message --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a550af31e7..e184596370 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1670,7 +1670,7 @@ async def _get_and_validate_existing_key( if existing_key_row is None: raise ProxyException( - message=f"Key not found. Passed key={token}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -4947,7 +4947,7 @@ async def block_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found. Passed key={data.key}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -5056,7 +5056,7 @@ async def unblock_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found. Passed key={data.key}", + message=f"Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, From 7b600cdbfe333333a3d86ea835f44b8f00ace195 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Mar 2026 16:31:50 -0700 Subject: [PATCH 54/57] linting --- .../proxy/management_endpoints/key_management_endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e184596370..831922ec3f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1670,7 +1670,7 @@ async def _get_and_validate_existing_key( if existing_key_row is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -4947,7 +4947,7 @@ async def block_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, @@ -5056,7 +5056,7 @@ async def unblock_key( ) if existing_record is None: raise ProxyException( - message=f"Key not found.", + message="Key not found.", type=ProxyErrorTypes.not_found_error, param="key", code=status.HTTP_404_NOT_FOUND, From 6f1bac07e57e4fcd9e2b9e83716f28acfc36462b Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 17:11:04 -0700 Subject: [PATCH 55/57] chore: apply black formatting to proxy/_types.py to fix lint CI --- litellm/proxy/_types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f5568757a2..d62bfbb7d5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2955,7 +2955,9 @@ class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): endTime: Union[str, datetime, None] -AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated"] +AUDIT_ACTIONS = Literal[ + "created", "updated", "deleted", "blocked", "unblocked", "rotated" +] class LiteLLM_AuditLogs(LiteLLMPydanticObjectBase): From e668ca310dabf5c56f0792e97f166711445cb09c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Mar 2026 00:28:21 +0000 Subject: [PATCH 56/57] docs: add LiteLLM license key environment variable instructions Added a new section to the config.yaml documentation explaining how to set the LITELLM_LICENSE environment variable for enterprise features. Co-authored-by: Krish Dholakia --- docs/my-website/docs/proxy/configs.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 56a8b9566d..84a6fac121 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -602,6 +602,22 @@ Since you shouldn't use 12.5, round down to **10** to leave a safety buffer. Thi - Total maximum connections: 8 workers × 10 connections = 80 connections - This stays safely under your database's 100 connection limit +## LiteLLM License Key (Enterprise) + +To enable [LiteLLM Enterprise features](https://docs.litellm.ai/docs/proxy/enterprise), set your license key as an environment variable: + +```bash +export LITELLM_LICENSE="eyJ..." +``` + +The license key is a JWT token provided when you purchase a LiteLLM Enterprise license. Once set, LiteLLM will automatically detect and activate enterprise features. + +You can also add it to your `.env` file: + +```env +LITELLM_LICENSE="eyJ..." +``` + ## Extras From 87b5039aaab7ad5cd2cfa72c160d49e470738a72 Mon Sep 17 00:00:00 2001 From: joereyna Date: Thu, 19 Mar 2026 18:11:40 -0700 Subject: [PATCH 57/57] chore: apply black formatting to fix lint CI (batch 3) --- litellm/integrations/langsmith.py | 18 +++-- litellm/litellm_core_utils/litellm_logging.py | 5 +- .../litellm_core_utils/streaming_handler.py | 2 +- litellm/llms/anthropic/chat/handler.py | 40 +++++------ litellm/llms/anthropic/chat/transformation.py | 67 ++++++++++++------- .../anthropic_passthrough_logging_handler.py | 28 ++++---- litellm/proxy/utils.py | 6 +- .../transformation.py | 6 +- litellm/responses/streaming_iterator.py | 12 ++-- litellm/types/llms/openai.py | 36 +++++----- 10 files changed, 121 insertions(+), 99 deletions(-) diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 479b5027ef..9cc3735992 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -114,7 +114,9 @@ class LangsmithLogger(CustomBatchLogger): self, metadata: dict, credentials: LangsmithCredentialsObject ): return { - "project_name": metadata.get("project_name", credentials["LANGSMITH_PROJECT"]), + "project_name": metadata.get( + "project_name", credentials["LANGSMITH_PROJECT"] + ), "run_name": metadata.get("run_name", self.langsmith_default_run_name), "run_id": metadata.get("id", metadata.get("run_id", None)), "parent_run_id": metadata.get("parent_run_id", None), @@ -132,7 +134,9 @@ class LangsmithLogger(CustomBatchLogger): extra_metadata[key] = requester_metadata[key] return extra_metadata - def _build_outputs_with_usage(self, payload: StandardLoggingPayload) -> Dict[str, Any]: + def _build_outputs_with_usage( + self, payload: StandardLoggingPayload + ) -> Dict[str, Any]: response = payload["response"] outputs: Dict[str, Any] if isinstance(response, dict): @@ -171,7 +175,7 @@ class LangsmithLogger(CustomBatchLogger): try: _litellm_params = kwargs.get("litellm_params", {}) or {} metadata = _litellm_params.get("metadata", {}) or {} - + fields = self._extract_metadata_fields(metadata, credentials) verbose_logger.debug( f"Langsmith Logging - project_name: {fields['project_name']}, run_name {fields['run_name']}" @@ -202,7 +206,13 @@ class LangsmithLogger(CustomBatchLogger): if payload["error_str"] is not None and payload["status"] == "failure": data["error"] = payload["error_str"] - for key in ("id", "parent_run_id", "trace_id", "session_id", "dotted_order"): + for key in ( + "id", + "parent_run_id", + "trace_id", + "session_id", + "dotted_order", + ): field_key = "run_id" if key == "id" else key if fields[field_key]: data[key] = fields[field_key] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ca865d73da..fea139a64b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3331,7 +3331,10 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, TextCompletionResponse): return result - elif isinstance(result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)): + elif isinstance( + result, + (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent), + ): ## return unified Usage object if isinstance(result.response.usage, ResponseAPIUsage): transformed_usage = ( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6e512a1e57..96e70845b2 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -279,7 +279,7 @@ class CustomStreamWrapper: model="", llm_provider="", ) - + def check_special_tokens(self, chunk: str, finish_reason: Optional[str]): """ Output parse / special tokens for sagemaker + hf streaming. diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index a2389f4429..9f2ddcae2c 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -578,7 +578,9 @@ class ModelResponseIterator: speed=self.speed, ) - def _content_block_delta_helper(self, chunk: dict) -> Tuple[ + def _content_block_delta_helper( + self, chunk: dict + ) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], @@ -803,9 +805,9 @@ class ModelResponseIterator: tool_input = content_block_start["content_block"].get( "input", {} ) - self._server_tool_inputs[self._current_server_tool_id] = ( - tool_input - ) + self._server_tool_inputs[ + self._current_server_tool_id + ] = tool_input # Include caller information if present (for programmatic tool calling) if "caller" in content_block_start["content_block"]: caller_data = content_block_start["content_block"]["caller"] @@ -826,9 +828,9 @@ class ModelResponseIterator: # Handle compaction blocks # The full content comes in content_block_start self.compaction_blocks.append(content_block_start["content_block"]) - provider_specific_fields["compaction_blocks"] = ( - self.compaction_blocks - ) + provider_specific_fields[ + "compaction_blocks" + ] = self.compaction_blocks provider_specific_fields["compaction_start"] = { "type": "compaction", "content": content_block_start["content_block"].get( @@ -850,9 +852,9 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields["web_search_results"] = ( - self.web_search_results - ) + provider_specific_fields[ + "web_search_results" + ] = self.web_search_results elif content_type == "web_fetch_tool_result": # Capture web_fetch_tool_result for multi-turn reconstruction # The full content comes in content_block_start, not in deltas @@ -860,18 +862,18 @@ class ModelResponseIterator: self.web_search_results.append( content_block_start["content_block"] ) - provider_specific_fields["web_search_results"] = ( - self.web_search_results - ) + provider_specific_fields[ + "web_search_results" + ] = self.web_search_results elif content_type != "tool_search_tool_result": # Handle other tool results (code execution, etc.) # Skip tool_search_tool_result as it's internal metadata self.tool_results.append(content_block_start["content_block"]) provider_specific_fields["tool_results"] = self.tool_results # Convert to provider-neutral code_interpreter_results - provider_specific_fields["code_interpreter_results"] = ( - self._build_code_interpreter_results() - ) + provider_specific_fields[ + "code_interpreter_results" + ] = self._build_code_interpreter_results() elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore @@ -928,9 +930,9 @@ class ModelResponseIterator: ) if container_id and self.tool_results: self._container_id = container_id - provider_specific_fields["code_interpreter_results"] = ( - self._build_code_interpreter_results() - ) + provider_specific_fields[ + "code_interpreter_results" + ] = self._build_code_interpreter_results() elif type_chunk == "message_start": """ Anthropic diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 808b68fefd..73d1b02c76 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -964,11 +964,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if mcp_servers: optional_params["mcp_servers"] = mcp_servers elif param == "tool_choice" or param == "parallel_tool_calls": - _tool_choice: Optional[AnthropicMessagesToolChoice] = ( - self._map_tool_choice( - tool_choice=non_default_params.get("tool_choice"), - parallel_tool_use=non_default_params.get("parallel_tool_calls"), - ) + _tool_choice: Optional[ + AnthropicMessagesToolChoice + ] = self._map_tool_choice( + tool_choice=non_default_params.get("tool_choice"), + parallel_tool_use=non_default_params.get("parallel_tool_calls"), ) if _tool_choice is not None: @@ -1066,9 +1066,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.map_openai_context_management_to_anthropic(value) ) if anthropic_context_management is not None: - optional_params["context_management"] = ( - anthropic_context_management - ) + optional_params[ + "context_management" + ] = anthropic_context_management elif param == "speed" and isinstance(value, str): # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value @@ -1142,9 +1142,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text=system_message_block["content"], ) if "cache_control" in system_message_block: - anthropic_system_message_content["cache_control"] = ( - system_message_block["cache_control"] - ) + anthropic_system_message_content[ + "cache_control" + ] = system_message_block["cache_control"] anthropic_system_message_list.append( anthropic_system_message_content ) @@ -1168,9 +1168,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) if "cache_control" in _content: - anthropic_system_message_content["cache_control"] = ( - _content["cache_control"] - ) + anthropic_system_message_content[ + "cache_control" + ] = _content["cache_control"] anthropic_system_message_list.append( anthropic_system_message_content @@ -1467,7 +1467,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _message - def extract_response_content(self, completion_response: dict) -> Tuple[ + def extract_response_content( + self, completion_response: dict + ) -> Tuple[ str, Optional[List[Any]], Optional[ @@ -1684,7 +1686,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return usage - def _build_code_by_id_map(self, tool_calls: List[ChatCompletionToolCallChunk]) -> Dict[str, str]: + def _build_code_by_id_map( + self, tool_calls: List[ChatCompletionToolCallChunk] + ) -> Dict[str, str]: code_by_id: Dict[str, str] = {} for tc in tool_calls: try: @@ -1698,7 +1702,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return code_by_id def _build_code_interpreter_results( - self, tool_results: List[Any], code_by_id: Dict[str, str], container_id: Optional[str] + self, + tool_results: List[Any], + code_by_id: Dict[str, str], + container_id: Optional[str], ) -> List[OutputCodeInterpreterCall]: code_interpreter_results = [] for tr in tool_results: @@ -1723,7 +1730,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self, completion_response: dict, citations: Optional[List[Any]], - thinking_blocks: Optional[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]], + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], web_search_results: Optional[List[Any]], tool_results: Optional[List[Any]], compaction_blocks: Optional[List[Any]], @@ -1733,14 +1744,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "citations": citations, "thinking_blocks": thinking_blocks, } - + context_management = completion_response.get("context_management") if context_management is not None: provider_specific_fields["context_management"] = context_management - + if web_search_results is not None: provider_specific_fields["web_search_results"] = web_search_results - + if tool_results is not None: provider_specific_fields["tool_results"] = tool_results container_id = ( @@ -1752,15 +1763,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): code_interpreter_results = self._build_code_interpreter_results( tool_results, code_by_id, container_id ) - provider_specific_fields["code_interpreter_results"] = code_interpreter_results - + provider_specific_fields[ + "code_interpreter_results" + ] = code_interpreter_results + container = completion_response.get("container") if container is not None: provider_specific_fields["container"] = container - + if compaction_blocks is not None: provider_specific_fields["compaction_blocks"] = compaction_blocks - + return provider_specific_fields def transform_parsed_response( @@ -1830,7 +1843,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _message = json_mode_message model_response.choices[0].message = _message - model_response._hidden_params["original_response"] = completion_response["content"] + model_response._hidden_params["original_response"] = completion_response[ + "content" + ] model_response.choices[0].finish_reason = cast( OpenAIChatCompletionFinishReason, map_finish_reason(completion_response["stop_reason"]), diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 3241c1ca93..fcb1e0b2e4 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -6,23 +6,21 @@ import httpx import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.litellm_logging import \ - use_custom_pricing_for_model +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.chat.handler import \ - ModelResponseIterator as AnthropicModelResponseIterator +from litellm.llms.anthropic.chat.handler import ( + ModelResponseIterator as AnthropicModelResponseIterator, +) from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body -from litellm.types.passthrough_endpoints.pass_through_endpoints import \ - PassthroughStandardLoggingPayload -from litellm.types.utils import (LiteLLMBatch, ModelResponse, - TextCompletionResponse) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + PassthroughStandardLoggingPayload, +) +from litellm.types.utils import LiteLLMBatch, ModelResponse, TextCompletionResponse if TYPE_CHECKING: - from litellm.types.passthrough_endpoints.pass_through_endpoints import \ - EndpointType + from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType from ..success_handler import PassThroughEndpointLogging else: @@ -333,8 +331,7 @@ class AnthropicPassthroughLoggingHandler: import base64 from litellm._uuid import uuid - from litellm.llms.anthropic.batches.transformation import \ - AnthropicBatchesConfig + from litellm.llms.anthropic.batches.transformation import AnthropicBatchesConfig from litellm.types.utils import Choices, SpecialEnums try: @@ -550,8 +547,7 @@ class AnthropicPassthroughLoggingHandler: managed_files_hook, "store_unified_object_id" ): # Create a mock user API key dict for the managed object storage - from litellm.proxy._types import (LitellmUserRoles, - UserAPIKeyAuth) + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth user_api_key_dict = UserAPIKeyAuth( user_id=kwargs.get("user_id", "default-user"), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 566e1ff517..351bc23915 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1898,9 +1898,9 @@ class ProxyLogging: normalized_call_type = CallTypes.aembedding.value if normalized_call_type is not None: litellm_logging_obj.call_type = normalized_call_type - litellm_logging_obj.model_call_details["call_type"] = ( - normalized_call_type - ) + litellm_logging_obj.model_call_details[ + "call_type" + ] = normalized_call_type # Pass-through endpoints are logged via the callback loop's # async_post_call_failure_hook — skip pre_call and failure handlers. if litellm_logging_obj.call_type == CallTypes.pass_through.value: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index cf18511bfa..b6479a3699 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2113,9 +2113,9 @@ class LiteLLMCompletionResponsesConfig: hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None ): - output_details_dict["reasoning_tokens"] = ( - completion_details.reasoning_tokens - ) + output_details_dict[ + "reasoning_tokens" + ] = completion_details.reasoning_tokens else: output_details_dict["reasoning_tokens"] = 0 diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8a91368dd6..10a74a5b3c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -168,14 +168,10 @@ class BaseResponsesAPIStreamingIterator: # Store the completed response (also for incomplete/failed so logging still fires) _chunk_type = getattr(openai_responses_api_chunk, "type", None) - if ( - openai_responses_api_chunk - and _chunk_type - in ( - ResponsesAPIStreamEvents.RESPONSE_COMPLETED, - ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, - ResponsesAPIStreamEvents.RESPONSE_FAILED, - ) + if openai_responses_api_chunk and _chunk_type in ( + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ResponsesAPIStreamEvents.RESPONSE_FAILED, ): self.completed_response = openai_responses_api_chunk # Add cost to usage object if include_cost_in_streaming_usage is True diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a265198e6b..5a80b40d61 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -970,12 +970,12 @@ class OpenAIChatCompletionChunk(ChatCompletionChunk): class Hyperparameters(BaseModel): batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." - learning_rate_multiplier: Optional[Union[str, float]] = ( - None # Scaling factor for the learning rate - ) - n_epochs: Optional[Union[str, int]] = ( - None # "The number of epochs to train the model for" - ) + learning_rate_multiplier: Optional[ + Union[str, float] + ] = None # Scaling factor for the learning rate + n_epochs: Optional[ + Union[str, int] + ] = None # "The number of epochs to train the model for" model_config = {"extra": "allow"} @@ -1004,18 +1004,18 @@ class FineTuningJobCreate(BaseModel): model: str # "The name of the model to fine-tune." training_file: str # "The ID of an uploaded file that contains training data." - hyperparameters: Optional[Hyperparameters] = ( - None # "The hyperparameters used for the fine-tuning job." - ) - suffix: Optional[str] = ( - None # "A string of up to 18 characters that will be added to your fine-tuned model name." - ) - validation_file: Optional[str] = ( - None # "The ID of an uploaded file that contains validation data." - ) - integrations: Optional[List[str]] = ( - None # "A list of integrations to enable for your fine-tuning job." - ) + hyperparameters: Optional[ + Hyperparameters + ] = None # "The hyperparameters used for the fine-tuning job." + suffix: Optional[ + str + ] = None # "A string of up to 18 characters that will be added to your fine-tuned model name." + validation_file: Optional[ + str + ] = None # "The ID of an uploaded file that contains validation data." + integrations: Optional[ + List[str] + ] = None # "A list of integrations to enable for your fine-tuning job." seed: Optional[int] = None # "The seed controls the reproducibility of the job."