From 08f0cbc2e939c6456feb2f7ef8edf118644748fa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 18 Mar 2026 21:36:39 -0700 Subject: [PATCH] fix: address greptile feedback --- litellm/litellm_core_utils/litellm_logging.py | 266 +++++++++++------- litellm/responses/streaming_iterator.py | 73 +++-- ...t_base_responses_api_streaming_iterator.py | 156 +++++++++- 3 files changed, 369 insertions(+), 126 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5e34a4c992..27cef85818 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,28 +12,47 @@ import time import traceback from datetime import datetime as dt_object from functools import lru_cache -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal, - Optional, Tuple, Type, Union, cast) +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Literal, + Optional, + Tuple, + Type, + Union, + cast, +) from httpx import Response from pydantic import BaseModel import litellm -from litellm import (_custom_logger_compatible_callbacks_literal, json_logs, - log_raw_request_response, turn_off_message_logging) +from litellm import ( + _custom_logger_compatible_callbacks_literal, + json_logs, + log_raw_request_response, + turn_off_message_logging, +) from litellm._logging import _is_debugging_on, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler -from litellm.constants import (DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, - DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - SENTRY_DENYLIST, SENTRY_PII_DENYLIST) -from litellm.cost_calculator import (RealtimeAPITokenUsageProcessor, - _select_model_name_for_cost_calc) +from litellm.constants import ( + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, + DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, + SENTRY_DENYLIST, + SENTRY_PII_DENYLIST, +) +from litellm.cost_calculator import ( + RealtimeAPITokenUsageProcessor, + _select_model_name_for_cost_calc, +) from litellm.integrations.agentops import AgentOps -from litellm.integrations.anthropic_cache_control_hook import \ - AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger @@ -42,48 +61,72 @@ from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger from litellm.litellm_core_utils.core_helpers import reconstruct_model_name from litellm.litellm_core_utils.get_litellm_params import get_litellm_params -from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import \ - StandardBuiltInToolCostTracking -from litellm.litellm_core_utils.logging_utils import \ - truncate_base64_in_messages +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, - redact_message_input_output_from_logging) + redact_message_input_output_from_logging, +) from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.llms.openai import (AllMessageValues, Batch, FineTuningJob, - HttpxBinaryResponseContent, - OpenAIFileObject, - OpenAIModerationResponse, - ResponseAPIUsage, - ResponseCompletedEvent, - ResponseFailedEvent, - ResponseIncompleteEvent, - ResponsesAPIResponse) +from litellm.types.llms.openai import ( + AllMessageValues, + Batch, + FineTuningJob, + HttpxBinaryResponseContent, + OpenAIFileObject, + OpenAIModerationResponse, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, +) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse from litellm.types.utils import ( - CachingDetails, CallTypes, CostBreakdown, CostResponseTypes, - CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, - EmbeddingResponse, GuardrailStatus, ImageResponse, LiteLLMBatch, - LiteLLMLoggingBaseClass, LiteLLMRealtimeStreamLoggingObject, ModelResponse, - ModelResponseStream, RawRequestTypedDict, StandardBuiltInToolsParams, - StandardCallbackDynamicParams, StandardLoggingAdditionalHeaders, - StandardLoggingHiddenParams, StandardLoggingMCPToolCall, - StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, - StandardLoggingModelInformation, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, StandardLoggingPayloadStatus, + CachingDetails, + CallTypes, + CostBreakdown, + CostResponseTypes, + CustomPricingLiteLLMParams, + DynamicPromptManagementParamLiteral, + EmbeddingResponse, + GuardrailStatus, + ImageResponse, + LiteLLMBatch, + LiteLLMLoggingBaseClass, + LiteLLMRealtimeStreamLoggingObject, + ModelResponse, + ModelResponseStream, + RawRequestTypedDict, + StandardBuiltInToolsParams, + StandardCallbackDynamicParams, + StandardLoggingAdditionalHeaders, + StandardLoggingHiddenParams, + StandardLoggingMCPToolCall, + StandardLoggingMetadata, + StandardLoggingModelCostFailureDebugInformation, + StandardLoggingModelInformation, + StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, + StandardLoggingPayloadStatus, StandardLoggingPayloadStatusFields, - StandardLoggingPromptManagementMetadata, StandardLoggingVectorStoreRequest, - TextCompletionResponse, TranscriptionResponse, Usage) + StandardLoggingPromptManagementMetadata, + StandardLoggingVectorStoreRequest, + TextCompletionResponse, + TranscriptionResponse, + Usage, +) from litellm.types.videos.main import VideoObject -from litellm.utils import (_get_base_model_from_metadata, executor, - print_verbose) +from litellm.utils import _get_base_model_from_metadata, executor, print_verbose from ..integrations.argilla import ArgillaLogger from ..integrations.arize.arize_phoenix import ArizePhoenixLogger @@ -105,8 +148,7 @@ from ..integrations.humanloop import HumanloopLogger from ..integrations.lago import LagoLogger from ..integrations.langfuse.langfuse import LangFuseLogger from ..integrations.langfuse.langfuse_handler import LangFuseHandler -from ..integrations.langfuse.langfuse_prompt_management import \ - LangfusePromptManagement +from ..integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement from ..integrations.langsmith import LangsmithLogger from ..integrations.litellm_agent import LiteLLMAgentModelResolver from ..integrations.literal_ai import LiteralAILogger @@ -121,30 +163,34 @@ from ..integrations.s3_v2 import S3Logger as S3V2Logger from ..integrations.supabase import Supabase from ..integrations.traceloop import TraceloopLogger from .exception_mapping_utils import _get_response_headers -from .initialize_dynamic_callback_params import \ - initialize_standard_callback_dynamic_params as \ - _initialize_standard_callback_dynamic_params +from .initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params as _initialize_standard_callback_dynamic_params, +) from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: - from litellm.llms.base_llm.passthrough.transformation import \ - BasePassthroughConfig + from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig try: - from litellm_enterprise.enterprise_callbacks.callback_controls import \ - EnterpriseCallbackControls - from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import \ - PagerDutyAlerting - from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import \ - ResendEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import \ - SendGridEmailLogger - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import \ - SMTPEmailLogger - from litellm_enterprise.litellm_core_utils.litellm_logging import \ - StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup + from litellm_enterprise.enterprise_callbacks.callback_controls import ( + EnterpriseCallbackControls, + ) + from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import ( + PagerDutyAlerting, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ( + ResendEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import ( + SendGridEmailLogger, + ) + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import ( + SMTPEmailLogger, + ) + from litellm_enterprise.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup as EnterpriseStandardLoggingPayloadSetup, + ) - from litellm.integrations.generic_api.generic_api_callback import \ - GenericAPILogger + from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger EnterpriseStandardLoggingPayloadSetupVAR: Optional[ Type[EnterpriseStandardLoggingPayloadSetup] @@ -1723,8 +1769,9 @@ class Logging(LiteLLMLoggingBaseClass): ) standard_logging_payload["response"] = response_dict elif isinstance(result, TranscriptionResponse): - from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import \ - TranscriptionUsageObjectTransformation + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, + ) result = result.model_copy() transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore @@ -2407,8 +2454,9 @@ class Logging(LiteLLMLoggingBaseClass): ): # polling job will query these frequently, don't spam db logs return - from litellm.proxy.openai_files_endpoints.common_utils import \ - _is_base64_encoded_unified_file_id + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + ) # check if file id is a unified file id is_base64_unified_file_id = _is_base64_encoded_unified_file_id(result.id) @@ -3305,7 +3353,6 @@ class Logging(LiteLLMLoggingBaseClass): return result.response else: return None - return None def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: """ @@ -3551,8 +3598,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 elif callback == "s3": s3Logger = S3Logger() elif callback == "wandb": - from litellm.integrations.weights_biases import \ - WeightsBiasesLogger + from litellm.integrations.weights_biases import WeightsBiasesLogger weightsBiasesLogger = WeightsBiasesLogger() elif callback == "logfire": @@ -3616,8 +3662,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_posthog_logger) return _posthog_logger # type: ignore elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -3738,7 +3783,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _opik_logger # type: ignore elif logging_integration == "arize": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_config = ArizeLogger.get_arize_config() if arize_config.endpoint is None: @@ -3765,7 +3812,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return _arize_otel_logger # type: ignore elif logging_integration == "arize_phoenix": from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) arize_phoenix_config = ArizePhoenixLogger.get_arize_phoenix_config() otel_config = OpenTelemetryConfig( @@ -3819,7 +3868,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "levo": from litellm.integrations.levo.levo import LevoLogger from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) levo_config = LevoLogger.get_levo_config() otel_config = OpenTelemetryConfig( @@ -3868,8 +3919,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(galileo_logger) return galileo_logger # type: ignore elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -3889,8 +3939,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(focus_logger) return focus_logger # type: ignore elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -3910,7 +3959,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 if "LOGFIRE_TOKEN" not in os.environ: raise ValueError("LOGFIRE_TOKEN not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) logfire_base_url = os.getenv( "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev" @@ -3928,8 +3979,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(_otel_logger) return _otel_logger # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): @@ -3951,8 +4003,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -3978,7 +4031,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 raise ValueError("LANGTRACE_API_KEY not found in environment variables") from litellm.integrations.opentelemetry import ( - OpenTelemetry, OpenTelemetryConfig) + OpenTelemetry, + OpenTelemetryConfig, + ) otel_config = OpenTelemetryConfig( exporter="otlp_http", @@ -4014,8 +4069,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(langfuse_logger) return langfuse_logger # type: ignore elif logging_integration == "langfuse_otel": - from litellm.integrations.langfuse.langfuse_otel import \ - LangfuseOtelLogger + from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger for callback in _in_memory_loggers: if ( @@ -4033,7 +4087,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "weave_otel": from litellm.integrations.opentelemetry import OpenTelemetryConfig from litellm.integrations.weave.weave_otel import ( - WeaveOtelLogger, get_weave_otel_config) + WeaveOtelLogger, + get_weave_otel_config, + ) weave_otel_config = get_weave_otel_config() @@ -4069,8 +4125,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(anthropic_cache_control_hook) return anthropic_cache_control_hook # type: ignore elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -4130,8 +4187,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(dotprompt_logger) return dotprompt_logger # type: ignore elif logging_integration == "bitbucket": - from litellm.integrations.bitbucket.bitbucket_prompt_manager import \ - BitBucketPromptManager + from litellm.integrations.bitbucket.bitbucket_prompt_manager import ( + BitBucketPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, BitBucketPromptManager): @@ -4148,8 +4206,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 _in_memory_loggers.append(bitbucket_logger) return bitbucket_logger # type: ignore elif logging_integration == "gitlab": - from litellm.integrations.gitlab.gitlab_prompt_manager import \ - GitLabPromptManager + from litellm.integrations.gitlab.gitlab_prompt_manager import ( + GitLabPromptManager, + ) for callback in _in_memory_loggers: if isinstance(callback, GitLabPromptManager): @@ -4237,8 +4296,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, OpenMeterLogger): return callback elif logging_integration == "braintrust": - from litellm.integrations.braintrust_logging import \ - BraintrustLogger + from litellm.integrations.braintrust_logging import BraintrustLogger for callback in _in_memory_loggers: if isinstance(callback, BraintrustLogger): @@ -4248,8 +4306,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, GalileoObserve): return callback elif logging_integration == "cloudzero": - from litellm.integrations.cloudzero.cloudzero import \ - CloudZeroLogger + from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): @@ -4263,8 +4320,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 ): # exact match; exclude subclasses like VantageLogger return callback elif logging_integration == "vantage": - from litellm.integrations.vantage.vantage_logger import \ - VantageLogger + from litellm.integrations.vantage.vantage_logger import VantageLogger for callback in _in_memory_loggers: if isinstance(callback, VantageLogger): @@ -4364,15 +4420,17 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback # type: ignore elif logging_integration == "dynamic_rate_limiter": - from litellm.proxy.hooks.dynamic_rate_limiter import \ - _PROXY_DynamicRateLimitHandler + from litellm.proxy.hooks.dynamic_rate_limiter import ( + _PROXY_DynamicRateLimitHandler, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore elif logging_integration == "dynamic_rate_limiter_v3": - from litellm.proxy.hooks.dynamic_rate_limiter_v3 import \ - _PROXY_DynamicRateLimitHandlerV3 + from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( + _PROXY_DynamicRateLimitHandlerV3, + ) for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3): @@ -4404,8 +4462,9 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 if isinstance(callback, AnthropicCacheControlHook): return callback elif logging_integration == "vector_store_pre_call_hook": - from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import \ - VectorStorePreCallHook + from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + VectorStorePreCallHook, + ) for callback in _in_memory_loggers: if isinstance(callback, VectorStorePreCallHook): @@ -5577,6 +5636,7 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal + # used for unit testing from typing import Any, Dict, List, Optional, Union diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a6a1074067..8a91368dd6 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -8,29 +8,31 @@ from typing import Any, Dict, List, Optional import httpx import litellm -from litellm.constants import (LITELLM_MAX_STREAMING_DURATION_SECONDS, - STREAM_SSE_DONE_STRING) +from litellm.constants import ( + LITELLM_MAX_STREAMING_DURATION_SECONDS, + STREAM_SSE_DONE_STRING, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.litellm_logging import \ - Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.llm_response_utils.get_api_base import \ - get_api_base -from litellm.litellm_core_utils.llm_response_utils.response_metadata import \ - update_response_metadata +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base +from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, +) from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.responses.transformation import \ - BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.llms.openai import (OutputTextDeltaEvent, ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIRequestParams, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, - ResponsesAPIStreamingResponse) +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, + ResponsesAPIStreamingResponse, +) from litellm.types.utils import CallTypes -from litellm.utils import (CustomStreamWrapper, - async_post_call_success_deployment_hook) +from litellm.utils import CustomStreamWrapper, async_post_call_success_deployment_hook class BaseResponsesAPIStreamingIterator: @@ -198,10 +200,12 @@ class BaseResponsesAPIStreamingIterator: if cost is not None: setattr(usage_obj, "cost", cost) except Exception: - # If cost calculation fails, continue without cost pass - self._handle_logging_completed_response() + if _chunk_type == ResponsesAPIStreamEvents.RESPONSE_FAILED: + self._handle_logging_failed_response() + else: + self._handle_logging_completed_response() return openai_responses_api_chunk @@ -219,6 +223,32 @@ class BaseResponsesAPIStreamingIterator: """Base implementation - should be overridden by subclasses""" pass + def _handle_logging_failed_response(self): + """ + Handle logging for RESPONSE_FAILED events by routing to failure handlers. + + Unlike _handle_logging_completed_response (which calls success handlers), + this constructs an exception from the response error and routes to + async_failure_handler / failure_handler so logging integrations correctly + record the call as failed. + """ + response_obj = ( + getattr(self.completed_response, "response", None) + if self.completed_response + else None + ) + error_info = getattr(response_obj, "error", None) if response_obj else None + error_message = "Response failed" + if isinstance(error_info, dict): + error_message = error_info.get("message", str(error_info)) + exception = litellm.APIError( + status_code=500, + message=error_message, + llm_provider=self.custom_llm_provider or "", + model=self.model or "", + ) + self._handle_failure(exception) + async def _call_post_streaming_deployment_hook(self, chunk): """ Allow callbacks to modify streaming chunks before returning (parity with chat). @@ -697,8 +727,7 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): # --------------------------------------------------------------------------- from litellm._logging import verbose_logger -from litellm.litellm_core_utils.thread_pool_executor import \ - executor as _ws_executor +from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor RESPONSES_WS_LOGGED_EVENT_TYPES = [ "response.created", diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 860445d875..e9181d810e 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -30,9 +30,11 @@ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterat from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, - OutputTextDeltaEvent + OutputTextDeltaEvent, ) @@ -429,3 +431,155 @@ class TestBaseResponsesAPIStreamingIterator: mock_logging_obj.async_failure_handler.assert_not_called() mock_logging_obj.failure_handler.assert_not_called() + def test_process_chunk_response_failed_calls_failure_handler(self): + """ + Test that a RESPONSE_FAILED event routes to failure handlers, + not success handlers. Failed responses represent genuine LLM-level + errors and should be logged as failures. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_failed_123" + mock_responses_api_response.error = { + "type": "server_error", + "message": "The model encountered an error", + } + mock_responses_api_response.usage = None + + mock_failed_event = Mock(spec=ResponseFailedEvent) + mock_failed_event.type = ResponsesAPIStreamEvents.RESPONSE_FAILED + mock_failed_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_failed_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.failed", + "response": { + "id": "resp_failed_123", + "error": { + "type": "server_error", + "message": "The model encountered an error", + }, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "litellm.responses.streaming_iterator.run_async_function" + ) as mock_run_async, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_FAILED + assert iterator.completed_response == result + + # Failure handler should have been called via _handle_failure + mock_run_async.assert_called_once() + call_kwargs = mock_run_async.call_args + assert ( + call_kwargs[1]["async_function"] + == mock_logging_obj.async_failure_handler + ) + + mock_executor.submit.assert_called_once() + submit_args = mock_executor.submit.call_args + assert submit_args[0][0] == mock_logging_obj.failure_handler + + def test_process_chunk_response_incomplete_calls_success_handler(self): + """ + Test that a RESPONSE_INCOMPLETE event routes to success handlers. + Incomplete responses (e.g. max_output_tokens reached) are still valid + responses with usage data — analogous to finish_reason='length' in chat. + """ + from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_lines = Mock() + mock_logging_obj = Mock(spec=LiteLLMLoggingObj) + mock_logging_obj.model_call_details = {"litellm_params": {}} + mock_logging_obj.async_failure_handler = Mock() + mock_logging_obj.failure_handler = Mock() + mock_logging_obj.async_success_handler = Mock() + mock_logging_obj.success_handler = Mock() + mock_config = Mock(spec=BaseResponsesAPIConfig) + + mock_responses_api_response = Mock(spec=ResponsesAPIResponse) + mock_responses_api_response.id = "resp_incomplete_123" + mock_responses_api_response.incomplete_details = { + "reason": "max_output_tokens" + } + mock_responses_api_response.usage = None + + mock_incomplete_event = Mock(spec=ResponseIncompleteEvent) + mock_incomplete_event.type = ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + mock_incomplete_event.response = mock_responses_api_response + + mock_config.transform_streaming_response.return_value = mock_incomplete_event + + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-4", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + litellm_metadata={"model_info": {"id": "model_123"}}, + custom_llm_provider="openai", + ) + + test_chunk_data = { + "type": "response.incomplete", + "response": { + "id": "resp_incomplete_123", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + } + + with patch.object( + ResponsesAPIRequestUtils, + "_update_responses_api_response_id_with_model_id", + return_value=mock_responses_api_response, + ), patch( + "asyncio.create_task" + ) as mock_create_task, patch( + "litellm.responses.streaming_iterator.executor" + ) as mock_executor: + result = iterator._process_chunk(json.dumps(test_chunk_data)) + + assert result is not None + assert result.type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + assert iterator.completed_response == result + + # Success handler should have been called (via _handle_logging_completed_response) + mock_create_task.assert_called_once() + mock_executor.submit.assert_called_once() + + # Failure handlers should NOT have been called + mock_logging_obj.async_failure_handler.assert_not_called() + mock_logging_obj.failure_handler.assert_not_called() +