diff --git a/enterprise/enterprise_hooks/session_handler.py b/enterprise/enterprise_hooks/session_handler.py new file mode 100644 index 0000000000..e3756c386b --- /dev/null +++ b/enterprise/enterprise_hooks/session_handler.py @@ -0,0 +1,136 @@ +from litellm.proxy._types import SpendLogsPayload +from litellm.integrations.custom_logger import CustomLogger +from litellm._logging import verbose_proxy_logger +from typing import Optional, List, Union +import json +from litellm.types.utils import ModelResponse, Message +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionResponseMessage, + GenericChatCompletionMessage, + ResponseInputParam, +) +from litellm.types.utils import ChatCompletionMessageToolCall + +from litellm.responses.utils import ResponsesAPIRequestUtils +from typing import TypedDict + +class ChatCompletionSession(TypedDict, total=False): + messages: List[Union[AllMessageValues, GenericChatCompletionMessage, ChatCompletionMessageToolCall, ChatCompletionResponseMessage, Message]] + litellm_session_id: Optional[str] + +class _ENTERPRISE_ResponsesSessionHandler: + @staticmethod + async def get_chat_completion_message_history_for_previous_response_id( + previous_response_id: str, + ) -> ChatCompletionSession: + """ + Return the chat completion message history for a previous response id + """ + from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig + all_spend_logs: List[SpendLogsPayload] = await _ENTERPRISE_ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(previous_response_id) + + litellm_session_id: Optional[str] = None + if len(all_spend_logs) > 0: + litellm_session_id = all_spend_logs[0].get("session_id") + + chat_completion_message_history: List[ + Union[ + AllMessageValues, + GenericChatCompletionMessage, + ChatCompletionMessageToolCall, + ChatCompletionResponseMessage, + Message, + ] + ] = [] + for spend_log in all_spend_logs: + proxy_server_request: Union[str, dict] = spend_log.get("proxy_server_request") or "{}" + proxy_server_request_dict: Optional[dict] = None + response_input_param: Optional[Union[str, ResponseInputParam]] = None + if isinstance(proxy_server_request, dict): + proxy_server_request_dict = proxy_server_request + else: + proxy_server_request_dict = json.loads(proxy_server_request) + + ############################################################ + # Add Input messages for this Spend Log + ############################################################ + if proxy_server_request_dict: + _response_input_param = proxy_server_request_dict.get("input", None) + if isinstance(_response_input_param, str): + response_input_param = _response_input_param + elif isinstance(_response_input_param, dict): + response_input_param = ResponseInputParam(**_response_input_param) + + if response_input_param: + chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=response_input_param, + responses_api_request=proxy_server_request_dict or {} + ) + chat_completion_message_history.extend(chat_completion_messages) + + ############################################################ + # Add Output messages for this Spend Log + ############################################################ + _response_output = spend_log.get("response", "{}") + if isinstance(_response_output, dict): + # transform `ChatCompletion Response` to `ResponsesAPIResponse` + model_response = ModelResponse(**_response_output) + for choice in model_response.choices: + if hasattr(choice, "message"): + chat_completion_message_history.append(choice.message) + + verbose_proxy_logger.debug("chat_completion_message_history %s", json.dumps(chat_completion_message_history, indent=4, default=str)) + return ChatCompletionSession( + messages=chat_completion_message_history, + litellm_session_id=litellm_session_id + ) + + @staticmethod + async def get_all_spend_logs_for_previous_response_id( + previous_response_id: str + ) -> List[SpendLogsPayload]: + """ + Get all spend logs for a previous response id + + + SQL query + + SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id + """ + from litellm.proxy.proxy_server import prisma_client + decoded_response_id = ResponsesAPIRequestUtils._decode_responses_api_response_id(previous_response_id) + previous_response_id = decoded_response_id.get("response_id", previous_response_id) + if prisma_client is None: + return [] + + query = """ + WITH matching_session AS ( + SELECT session_id + FROM "LiteLLM_SpendLogs" + WHERE request_id = $1 + ) + SELECT * + FROM "LiteLLM_SpendLogs" + WHERE session_id IN (SELECT session_id FROM matching_session) + ORDER BY "endTime" ASC; + """ + + spend_logs = await prisma_client.db.query_raw( + query, + previous_response_id + ) + + verbose_proxy_logger.debug( + "Found the following spend logs for previous response id %s: %s", + previous_response_id, + json.dumps(spend_logs, indent=4, default=str) + ) + + + return spend_logs + + + + + diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250425182129_add_session_id/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250425182129_add_session_id/migration.sql new file mode 100644 index 0000000000..751c75e5f2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250425182129_add_session_id/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "proxy_server_request" JSONB DEFAULT '{}', +ADD COLUMN "session_id" TEXT; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 68e9382d75..42bb068b91 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -226,6 +226,8 @@ model LiteLLM_SpendLogs { requester_ip_address String? messages Json? @default("{}") response Json? @default("{}") + session_id String? + proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) } diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 77d4fd7d5d..9f77d5b362 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -28,7 +28,6 @@ from litellm._logging import _is_debugging_on, verbose_logger from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache from litellm.caching.caching_handler import LLMCachingHandler - from litellm.constants import ( DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, @@ -249,7 +248,7 @@ class Logging(LiteLLMLoggingBaseClass): self.start_time = start_time # log the call start time self.call_type = call_type self.litellm_call_id = litellm_call_id - self.litellm_trace_id = litellm_trace_id + self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4()) self.function_id = function_id self.streaming_chunks: List[Any] = [] # for generating complete stream response self.sync_streaming_chunks: List[Any] = ( @@ -3500,6 +3499,21 @@ class StandardLoggingPayloadSetup: else: return end_time_float - start_time_float + @staticmethod + def _get_standard_logging_payload_trace_id( + logging_obj: Logging, + litellm_params: dict, + ) -> str: + """ + Returns the `litellm_trace_id` for this request + + This helps link sessions when multiple requests are made in a single session + """ + dynamic_trace_id = litellm_params.get("litellm_trace_id") + if dynamic_trace_id: + return str(dynamic_trace_id) + return logging_obj.litellm_trace_id + def get_standard_logging_object_payload( kwargs: Optional[dict], @@ -3652,7 +3666,10 @@ def get_standard_logging_object_payload( payload: StandardLoggingPayload = StandardLoggingPayload( id=str(id), - trace_id=kwargs.get("litellm_trace_id"), # type: ignore + trace_id=StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id( + logging_obj=logging_obj, + litellm_params=litellm_params, + ), call_type=call_type or "", cache_hit=cache_hit, stream=stream, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 63c1a99cb1..8b148e3059 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -654,9 +654,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase): allowed_cache_controls: Optional[list] = [] config: Optional[dict] = {} permissions: Optional[dict] = {} - model_max_budget: Optional[ - dict - ] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} + model_max_budget: Optional[dict] = ( + {} + ) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {} model_config = ConfigDict(protected_namespaces=()) model_rpm_limit: Optional[dict] = None @@ -918,12 +918,12 @@ class NewCustomerRequest(BudgetNewRequest): alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) @model_validator(mode="before") @classmethod @@ -945,12 +945,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase): blocked: bool = False # allow/disallow requests for this end-user max_budget: Optional[float] = None budget_id: Optional[str] = None # give either a budget_id or max_budget - allowed_model_region: Optional[ - AllowedModelRegion - ] = None # require all user requests to use models in this specific region - default_model: Optional[ - str - ] = None # if no equivalent model in allowed region - default all requests to this model + allowed_model_region: Optional[AllowedModelRegion] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) class DeleteCustomerRequest(LiteLLMPydanticObjectBase): @@ -1086,9 +1086,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase): class AddTeamCallback(LiteLLMPydanticObjectBase): callback_name: str - callback_type: Optional[ - Literal["success", "failure", "success_and_failure"] - ] = "success_and_failure" + callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = ( + "success_and_failure" + ) callback_vars: Dict[str, str] @model_validator(mode="before") @@ -1346,9 +1346,9 @@ class ConfigList(LiteLLMPydanticObjectBase): stored_in_db: Optional[bool] field_default_value: Any premium_field: bool = False - nested_fields: Optional[ - List[FieldDetail] - ] = None # For nested dictionary or Pydantic fields + nested_fields: Optional[List[FieldDetail]] = ( + None # For nested dictionary or Pydantic fields + ) class ConfigGeneralSettings(LiteLLMPydanticObjectBase): @@ -1616,9 +1616,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase): budget_id: Optional[str] = None created_at: datetime updated_at: datetime - user: Optional[ - Any - ] = None # You might want to replace 'Any' with a more specific type if available + user: Optional[Any] = ( + None # You might want to replace 'Any' with a more specific type if available + ) litellm_budget_table: Optional[LiteLLM_BudgetTable] = None model_config = ConfigDict(protected_namespaces=()) @@ -2015,6 +2015,8 @@ class SpendLogsPayload(TypedDict): custom_llm_provider: Optional[str] messages: Optional[Union[str, list, dict]] response: Optional[Union[str, list, dict]] + proxy_server_request: Optional[str] + session_id: Optional[str] class SpanAttributes(str, enum.Enum): @@ -2366,9 +2368,9 @@ class TeamModelDeleteRequest(BaseModel): # Organization Member Requests class OrganizationMemberAddRequest(OrgMemberAddRequest): organization_id: str - max_budget_in_organization: Optional[ - float - ] = None # Users max budget within the organization + max_budget_in_organization: Optional[float] = ( + None # Users max budget within the organization + ) class OrganizationMemberDeleteRequest(MemberDeleteRequest): @@ -2557,9 +2559,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase): Maps provider names to their budget configs. """ - providers: Dict[ - str, ProviderBudgetResponseObject - ] = {} # Dictionary mapping provider names to their budget configurations + providers: Dict[str, ProviderBudgetResponseObject] = ( + {} + ) # Dictionary mapping provider names to their budget configurations class ProxyStateVariables(TypedDict): @@ -2687,9 +2689,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase): enforce_rbac: bool = False roles_jwt_field: Optional[str] = None # v2 on role mappings role_mappings: Optional[List[RoleMapping]] = None - object_id_jwt_field: Optional[ - str - ] = None # can be either user / team, inferred from the role mapping + object_id_jwt_field: Optional[str] = ( + None # can be either user / team, inferred from the role mapping + ) scope_mappings: Optional[List[ScopeMapping]] = None enforce_scope_based_access: bool = False enforce_team_based_model_access: bool = False diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 75c49211e7..78cb7e010a 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,8 +1,7 @@ model_list: - - model_name: openai/* + - model_name: anthropic/* litellm_params: - model: openai/* - api_key: os.environ/OPENAI_API_KEY - -router_settings: - optional_pre_call_checks: ["responses_api_deployment_check"] + model: anthropic/* + api_key: os.environ/ANTHROPIC_API_KEY +general_settings: + store_prompts_in_spend_logs: true diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 68e9382d75..42bb068b91 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -226,6 +226,8 @@ model LiteLLM_SpendLogs { requester_ip_address String? messages Json? @default("{}") response Json? @default("{}") + session_id String? + proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) } diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 70370f3b53..bb3402f260 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2858,3 +2858,47 @@ async def ui_get_spend_by_tags( ) return {"spend_per_tag": ui_tags} + + +@router.get( + "/spend/logs/session/ui", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, + responses={ + 200: {"model": List[LiteLLM_SpendLogs]}, + }, +) +async def ui_view_session_spend_logs( + session_id: str = fastapi.Query( + description="Get all spend logs for a particular session", + ), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get all spend logs for a particular session + """ + from litellm.proxy.proxy_server import prisma_client + + try: + if prisma_client is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Database not connected", + ) + + # Build query conditions + where_conditions = {"session_id": session_id} + # Query the database + result = await prisma_client.db.litellm_spendlogs.find_many( + where=where_conditions, order={"startTime": "asc"} + ) + return result + except Exception as e: + if isinstance(e, HTTPException): + raise e + else: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e), + ) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index ecd3963a92..b6046f1fa3 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -137,9 +137,6 @@ def get_logging_payload( # noqa: PLR0915 # standardize this function to be used across, s3, dynamoDB, langfuse logging litellm_params = kwargs.get("litellm_params", {}) metadata = get_litellm_metadata_from_kwargs(kwargs) - metadata = _add_proxy_server_request_to_metadata( - metadata=metadata, litellm_params=litellm_params - ) completion_start_time = kwargs.get("completion_start_time", end_time) call_type = kwargs.get("call_type") cache_hit = kwargs.get("cache_hit", False) @@ -290,6 +287,14 @@ def get_logging_payload( # noqa: PLR0915 standard_logging_payload=standard_logging_payload, metadata=metadata ), response=_get_response_for_spend_logs_payload(standard_logging_payload), + proxy_server_request=_get_proxy_server_request_for_spend_logs_payload( + metadata=metadata, litellm_params=litellm_params + ), + session_id=( + standard_logging_payload.get("trace_id") + if standard_logging_payload is not None + else None + ), ) verbose_proxy_logger.debug( @@ -427,10 +432,10 @@ def _sanitize_request_body_for_spend_logs_payload( return {k: _sanitize_value(v) for k, v in request_body.items()} -def _add_proxy_server_request_to_metadata( +def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, litellm_params: dict, -) -> dict: +) -> str: """ Only store if _should_store_prompts_and_responses_in_spend_logs() is True """ @@ -442,8 +447,8 @@ def _add_proxy_server_request_to_metadata( _request_body = _proxy_server_request.get("body", {}) or {} _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) _request_body_json_str = json.dumps(_request_body, default=str) - metadata["proxy_server_request"] = _request_body_json_str - return metadata + return _request_body_json_str + return "{}" def _get_response_for_spend_logs_payload( diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 3580fe5e44..f960092f2b 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -89,6 +89,16 @@ class LiteLLMCompletionTransformationHandler: responses_api_request: ResponsesAPIOptionalRequestParams, **kwargs, ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: + + previous_response_id: Optional[str] = responses_api_request.get( + "previous_response_id" + ) + if previous_response_id: + litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id=previous_response_id, + litellm_completion_request=litellm_completion_request, + ) + litellm_completion_response: Union[ ModelResponse, litellm.CustomStreamWrapper ] = await litellm.acompletion( diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py deleted file mode 100644 index b114611c26..0000000000 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Responses API has previous_response_id, which is the id of the previous response. - -LiteLLM needs to maintain a cache of the previous response input, output, previous_response_id, and model. - -This class handles that cache. -""" - -from typing import List, Optional, Tuple, Union - -from typing_extensions import TypedDict - -from litellm.caching import InMemoryCache -from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse - -RESPONSES_API_PREVIOUS_RESPONSES_CACHE = InMemoryCache() -MAX_PREV_SESSION_INPUTS = 50 - - -class ResponsesAPISessionElement(TypedDict, total=False): - input: Union[str, ResponseInputParam] - output: ResponsesAPIResponse - response_id: str - previous_response_id: Optional[str] - - -class SessionHandler: - - def add_completed_response_to_cache( - self, response_id: str, session_element: ResponsesAPISessionElement - ): - RESPONSES_API_PREVIOUS_RESPONSES_CACHE.set_cache( - key=response_id, value=session_element - ) - - def get_chain_of_previous_input_output_pairs( - self, previous_response_id: str - ) -> List[Tuple[ResponseInputParam, ResponsesAPIResponse]]: - response_api_inputs: List[Tuple[ResponseInputParam, ResponsesAPIResponse]] = [] - current_previous_response_id = previous_response_id - - count_session_elements = 0 - while current_previous_response_id: - if count_session_elements > MAX_PREV_SESSION_INPUTS: - break - session_element = RESPONSES_API_PREVIOUS_RESPONSES_CACHE.get_cache( - key=current_previous_response_id - ) - if session_element: - response_api_inputs.append( - (session_element.get("input"), session_element.get("output")) - ) - current_previous_response_id = session_element.get( - "previous_response_id" - ) - else: - break - count_session_elements += 1 - return response_api_inputs diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 767778796b..719fc204a6 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -6,12 +6,12 @@ from typing import Any, Dict, List, Optional, Union from openai.types.responses.tool_param import FunctionToolParam +from enterprise.enterprise_hooks.session_handler import ( + ChatCompletionSession, + _ENTERPRISE_ResponsesSessionHandler, +) from litellm.caching import InMemoryCache from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.responses.litellm_completion_transformation.session_handler import ( - ResponsesAPISessionElement, - SessionHandler, -) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionResponseMessage, @@ -48,7 +48,6 @@ from litellm.types.utils import ( ########### Initialize Classes used for Responses API ########### TOOL_CALLS_CACHE = InMemoryCache() -RESPONSES_API_SESSION_HANDLER = SessionHandler() ########### End of Initialize Classes used for Responses API ########### @@ -90,7 +89,6 @@ class LiteLLMCompletionResponsesConfig: "messages": LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input, responses_api_request=responses_api_request, - previous_response_id=responses_api_request.get("previous_response_id"), ), "model": model, "tool_choice": responses_api_request.get("tool_choice"), @@ -131,14 +129,14 @@ class LiteLLMCompletionResponsesConfig: @staticmethod def transform_responses_api_input_to_messages( input: Union[str, ResponseInputParam], - responses_api_request: ResponsesAPIOptionalRequestParams, - previous_response_id: Optional[str] = None, + responses_api_request: Union[ResponsesAPIOptionalRequestParams, dict], ) -> List[ Union[ AllMessageValues, GenericChatCompletionMessage, ChatCompletionMessageToolCall, ChatCompletionResponseMessage, + Message, ] ]: """ @@ -150,6 +148,7 @@ class LiteLLMCompletionResponsesConfig: GenericChatCompletionMessage, ChatCompletionMessageToolCall, ChatCompletionResponseMessage, + Message, ] ] = [] if responses_api_request.get("instructions"): @@ -159,24 +158,6 @@ class LiteLLMCompletionResponsesConfig: ) ) - if previous_response_id: - previous_response_pairs = ( - RESPONSES_API_SESSION_HANDLER.get_chain_of_previous_input_output_pairs( - previous_response_id=previous_response_id - ) - ) - if previous_response_pairs: - for previous_response_pair in previous_response_pairs: - chat_completion_input_messages = LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=previous_response_pair[0], - ) - chat_completion_output_messages = LiteLLMCompletionResponsesConfig._transform_responses_api_outputs_to_chat_completion_messages( - responses_api_output=previous_response_pair[1], - ) - - messages.extend(chat_completion_input_messages) - messages.extend(chat_completion_output_messages) - messages.extend( LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( input=input, @@ -185,6 +166,29 @@ class LiteLLMCompletionResponsesConfig: return messages + @staticmethod + async def async_responses_api_session_handler( + previous_response_id: str, + litellm_completion_request: dict, + ) -> dict: + """ + Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages + """ + chat_completion_session: ChatCompletionSession = ChatCompletionSession( + messages=[], litellm_session_id=None + ) + if previous_response_id: + chat_completion_session = await _ENTERPRISE_ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + previous_response_id=previous_response_id + ) + _messages = litellm_completion_request.get("messages") or [] + session_messages = chat_completion_session.get("messages") or [] + litellm_completion_request["messages"] = session_messages + _messages + litellm_completion_request["litellm_trace_id"] = chat_completion_session.get( + "litellm_session_id" + ) + return litellm_completion_request + @staticmethod def _transform_response_input_param_to_chat_completion_message( input: Union[str, ResponseInputParam], @@ -471,11 +475,13 @@ class LiteLLMCompletionResponsesConfig: def transform_chat_completion_response_to_responses_api_response( request_input: Union[str, ResponseInputParam], responses_api_request: ResponsesAPIOptionalRequestParams, - chat_completion_response: ModelResponse, + chat_completion_response: Union[ModelResponse, dict], ) -> ResponsesAPIResponse: """ Transform a Chat Completion response into a Responses API response """ + if isinstance(chat_completion_response, dict): + chat_completion_response = ModelResponse(**chat_completion_response) responses_api_response: ResponsesAPIResponse = ResponsesAPIResponse( id=chat_completion_response.id, created_at=chat_completion_response.created, @@ -513,16 +519,6 @@ class LiteLLMCompletionResponsesConfig: ), user=getattr(chat_completion_response, "user", None), ) - - RESPONSES_API_SESSION_HANDLER.add_completed_response_to_cache( - response_id=responses_api_response.id, - session_element=ResponsesAPISessionElement( - input=request_input, - output=responses_api_response, - response_id=responses_api_response.id, - previous_response_id=responses_api_request.get("previous_response_id"), - ), - ) return responses_api_response @staticmethod diff --git a/schema.prisma b/schema.prisma index 68e9382d75..42bb068b91 100644 --- a/schema.prisma +++ b/schema.prisma @@ -226,6 +226,8 @@ model LiteLLM_SpendLogs { requester_ip_address String? messages Json? @default("{}") response Json? @default("{}") + session_id String? + proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) } diff --git a/tests/litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 181ca59aac..96eb254ed1 100644 --- a/tests/litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -22,6 +22,7 @@ from litellm.router import Router ignored_keys = [ "request_id", + "session_id", "startTime", "endTime", "completionStartTime", @@ -452,6 +453,7 @@ class TestSpendLogsPayload: expected_payload = SpendLogsPayload( **{ "request_id": "chatcmpl-34df56d5-4807-45c1-bb99-61e52586b802", + "session_id": "1234567890", "call_type": "acompletion", "api_key": "", "cache_hit": "None", @@ -482,6 +484,7 @@ class TestSpendLogsPayload: "custom_llm_provider": "openai", "messages": "{}", "response": "{}", + "proxy_server_request": "{}", } ) @@ -572,6 +575,7 @@ class TestSpendLogsPayload: "custom_llm_provider": "anthropic", "messages": "{}", "response": "{}", + "proxy_server_request": "{}", } ) @@ -660,6 +664,7 @@ class TestSpendLogsPayload: "custom_llm_provider": "anthropic", "messages": "{}", "response": "{}", + "proxy_server_request": "{}", } ) diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index 1157d6919e..8d0815ff69 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -1,5 +1,6 @@ { "request_id": "chatcmpl-2283081b-dc89-41f6-93e6-d4f914774027", + "session_id": "1234567890", "call_type": "acompletion", "api_key": "", "cache_hit": "None", @@ -23,5 +24,6 @@ "requester_ip_address": null, "custom_llm_provider": "openai", "messages": "{}", - "response": "{}" + "response": "{}", + "proxy_server_request": "{}" } \ No newline at end of file diff --git a/tests/logging_callback_tests/test_gcs_pub_sub.py b/tests/logging_callback_tests/test_gcs_pub_sub.py index 92ccb2ff9b..f231d01d3f 100644 --- a/tests/logging_callback_tests/test_gcs_pub_sub.py +++ b/tests/logging_callback_tests/test_gcs_pub_sub.py @@ -31,6 +31,7 @@ verbose_logger.setLevel(logging.DEBUG) ignored_keys = [ "request_id", + "session_id", "startTime", "endTime", "completionStartTime", @@ -170,7 +171,7 @@ def assert_gcs_pubsub_request_matches_expected_standard_logging_payload( "response_time", "completion_tokens", "prompt_tokens", - "total_tokens", + "total_tokens" ] for field in FIELDS_EXISTENCE_CHECKS: diff --git a/tests/logging_callback_tests/test_spend_logs.py b/tests/logging_callback_tests/test_spend_logs.py index d592931f25..5eed597160 100644 --- a/tests/logging_callback_tests/test_spend_logs.py +++ b/tests/logging_callback_tests/test_spend_logs.py @@ -385,10 +385,9 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch): assert payload["response"] == json.dumps( {"role": "assistant", "content": "Hi there!"} ) - parsed_metadata = json.loads(payload["metadata"]) - assert parsed_metadata["proxy_server_request"] == json.dumps( - {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello!"}]} - ) + proxy_server_request = json.loads(payload["proxy_server_request"] or "{}") + assert proxy_server_request["model"] == "gpt-4" + assert proxy_server_request["messages"] == [{"role": "user", "content": "Hello!"}] # Clean up - reset general_settings general_settings["store_prompts_in_spend_logs"] = False diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index 07871d3eea..d653c6c831 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -328,6 +328,48 @@ def test_get_final_response_obj(): litellm.turn_off_message_logging = False +def test_get_standard_logging_payload_trace_id(): + """Test _get_standard_logging_payload_trace_id with different input scenarios""" + # Test case 1: When litellm_trace_id is provided in litellm_params + from unittest.mock import MagicMock + + # Create a mock Logging object + mock_logging_obj = MagicMock() + mock_logging_obj.litellm_trace_id = "default-trace-id" + + # Test when litellm_trace_id is in litellm_params + litellm_params = {"litellm_trace_id": "dynamic-trace-id"} + result = StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id( + logging_obj=mock_logging_obj, + litellm_params=litellm_params + ) + assert result == "dynamic-trace-id" + + # Test case 2: When litellm_trace_id is not provided in litellm_params + litellm_params = {} + result = StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id( + logging_obj=mock_logging_obj, + litellm_params=litellm_params + ) + assert result == "default-trace-id" + + # Test case 3: When litellm_params is None + result = StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id( + logging_obj=mock_logging_obj, + litellm_params={} + ) + assert result == "default-trace-id" + + # Test case 4: When litellm_trace_id in params is not a string + litellm_params = {"litellm_trace_id": 12345} + result = StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id( + logging_obj=mock_logging_obj, + litellm_params=litellm_params + ) + assert result == "12345" + assert isinstance(result, str) + + def test_truncate_standard_logging_payload(): """ 1. original messages, response, and error_str should NOT BE MODIFIED, since these are from kwargs diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index a502716315..416c39e9c8 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4746,4 +4746,38 @@ export const teamPermissionsUpdateCall = async ( console.error("Failed to update team permissions:", error); throw error; } +}; + +/** + * Get all spend logs for a particular session + */ +export const sessionSpendLogsCall = async ( + accessToken: string, + session_id: string +) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/spend/logs/session/ui?session_id=${encodeURIComponent(session_id)}` + : `/spend/logs/session/ui?session_id=${encodeURIComponent(session_id)}`; + + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + handleError(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + return data; + } catch (error) { + console.error("Failed to fetch session logs:", error); + throw error; + } }; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx b/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx new file mode 100644 index 0000000000..089b28490f --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/SessionView.tsx @@ -0,0 +1,82 @@ +import React, { useState } from 'react'; +import { LogEntry } from './columns'; +import { DataTable } from './table'; +import { columns } from './columns'; +import { Card, Title, Text, Metric, AreaChart } from '@tremor/react'; +import { RequestViewer } from './index'; + +interface SessionViewProps { + sessionId: string; + logs: LogEntry[]; + onBack: () => void; +} + +export const SessionView: React.FC = ({ sessionId, logs, onBack }) => { + // Track which log row is expanded + const [expandedRequestId, setExpandedRequestId] = useState(null); + // Calculate session metrics + const totalCost = logs.reduce((sum, log) => sum + (log.spend || 0), 0); + const totalTokens = logs.reduce((sum, log) => sum + (log.total_tokens || 0), 0); + const startTime = logs.length > 0 ? new Date(logs[0].startTime) : new Date(); + const endTime = logs.length > 0 ? new Date(logs[logs.length - 1].endTime) : new Date(); + const durationMs = endTime.getTime() - startTime.getTime(); + const durationSec = (durationMs / 1000).toFixed(2); + + // Prepare data for the timeline chart + const timelineData = logs.map(log => ({ + time: new Date(log.startTime).toISOString(), + tokens: log.total_tokens || 0, + cost: log.spend || 0, + })); + + return ( +
+ {/* Header with back button */} +
+
+ +
+
+

Session Details

+

{sessionId}

+
+
+ + {/* Session Overview Cards */} +
+ + Total Requests + {logs.length} + + + Total Cost + ${totalCost.toFixed(4)} + + + Total Tokens + {totalTokens} + +
+ {/* Request Timeline */} + Session Logs +
+ true} + expandedRequestId={expandedRequestId} + onRowExpand={setExpandedRequestId} + /> +
+
+ ); +}; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 2732fdaa77..8b7fa0d1c1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -45,7 +45,10 @@ export type LogEntry = { requester_ip_address?: string; messages: string | any[] | Record; response: string | any[] | Record; + proxy_server_request?: string | any[] | Record; + session_id?: string; onKeyHashClick?: (keyHash: string) => void; + onSessionClick?: (sessionId: string) => void; }; export const columns: ColumnDef[] = [ @@ -119,6 +122,27 @@ export const columns: ColumnDef[] = [ ); }, }, + { + header: "Session ID", + accessorKey: "session_id", + cell: (info: any) => { + const value = String(info.getValue() || ""); + const onSessionClick = info.row.original.onSessionClick; + return ( + + + + ); + }, + }, + { header: "Request ID", accessorKey: "request_id", diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index fe82ac4ad8..953cd6840c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -3,7 +3,7 @@ import { useQuery } from "@tanstack/react-query"; import { useState, useRef, useEffect } from "react"; import { useQueryClient } from "@tanstack/react-query"; -import { uiSpendLogsCall, keyInfoV1Call } from "../networking"; +import { uiSpendLogsCall, keyInfoV1Call, sessionSpendLogsCall } from "../networking"; import { DataTable } from "./table"; import { columns, LogEntry } from "./columns"; import { Row } from "@tanstack/react-table"; @@ -15,6 +15,8 @@ import { ConfigInfoMessage } from './ConfigInfoMessage'; import { Tooltip } from "antd"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import KeyInfoView from "../key_info_view"; +import { SessionView } from './SessionView'; + interface SpendLogsTableProps { accessToken: string | null; token: string | null; @@ -75,6 +77,7 @@ export default function SpendLogsTable({ ); const [expandedRequestId, setExpandedRequestId] = useState(null); + const [selectedSessionId, setSelectedSessionId] = useState(null); const queryClient = useQueryClient(); @@ -201,6 +204,24 @@ export default function SpendLogsTable({ refetchIntervalInBackground: true, }); + // Fetch logs for a session if selected + const sessionLogs = useQuery({ + queryKey: ["sessionLogs", selectedSessionId], + queryFn: async () => { + if (!accessToken || !selectedSessionId) return { data: [], total: 0, page: 1, page_size: 50, total_pages: 1 }; + const response = await sessionSpendLogsCall(accessToken, selectedSessionId); + // If the API returns an array, wrap it in the same shape as PaginatedResponse + return { + data: response.data || response || [], + total: (response.data || response || []).length, + page: 1, + page_size: 1000, + total_pages: 1, + }; + }, + enabled: !!accessToken && !!selectedSessionId, + }); + // Add this effect to preserve expanded state when data refreshes useEffect(() => { if (logs.data?.data && expandedRequestId) { @@ -233,7 +254,18 @@ export default function SpendLogsTable({ }).map(log => ({ ...log, - onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash) + onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash), + onSessionClick: (sessionId: string) => { + if (sessionId) setSelectedSessionId(sessionId); + }, + })) || []; + + // For session logs, add onKeyHashClick/onSessionClick as well + const sessionData = + sessionLogs.data?.data?.map(log => ({ + ...log, + onKeyHashClick: (keyHash: string) => setSelectedKeyIdInfoView(keyHash), + onSessionClick: (sessionId: string) => {}, })) || []; // Add this function to handle manual refresh @@ -265,48 +297,66 @@ export default function SpendLogsTable({ setExpandedRequestId(requestId); }; + // When a session is selected, render the SessionView component + if (selectedSessionId && sessionLogs.data) { + return ( +
+ setSelectedSessionId(null)} + /> +
+ ); + } + return (
-

Request Logs

+

+ {selectedSessionId ? ( + <> + Session: {selectedSessionId} + + + ) : ( + "Request Logs" + )} +

{selectedKeyInfo && selectedKeyIdInfoView && selectedKeyInfo.api_key === selectedKeyIdInfoView ? ( setSelectedKeyIdInfoView(null)} /> + ) : selectedSessionId ? ( +
+ true} + // Optionally: add session-specific row expansion state + /> +
) : ( - <> -
-
-
-
-
- setSearchTerm(e.target.value)} - /> - - +
+
+
+
+
+ setSearchTerm(e.target.value)} /> - -
-
- - - {showFilters && ( -
-
-
- Where -
- - {showColumnDropdown && ( -
- {["Team ID", "Key Hash"].map((option) => ( - - ))} -
- )} -
- { - if (selectedFilter === "Team ID") { - setTempTeamId(e.target.value); - } else { - setTempKeyHash(e.target.value); - } - }} - /> - -
- -
- - -
-
-
- )} -
- -
-
+
+
- {quickSelectOpen && ( -
-
- {[ - { label: "Last 15 Minutes", value: 15, unit: "minutes" }, - { label: "Last Hour", value: 1, unit: "hours" }, - { label: "Last 4 Hours", value: 4, unit: "hours" }, - { label: "Last 24 Hours", value: 24, unit: "hours" }, - { label: "Last 7 Days", value: 7, unit: "days" }, - ].map((option) => ( + {showFilters && ( +
+
+
+ Where +
+ + {showColumnDropdown && ( +
+ {["Team ID", "Key Hash"].map((option) => ( + + ))} +
+ )} +
+ { + if (selectedFilter === "Team ID") { + setTempTeamId(e.target.value); + } else { + setTempKeyHash(e.target.value); + } + }} + /> - ))} -
- +
+ +
+ + +
)}
- - + + {quickSelectOpen && ( +
+
+ {[ + { label: "Last 15 Minutes", value: 15, unit: "minutes" }, + { label: "Last Hour", value: 1, unit: "hours" }, + { label: "Last 4 Hours", value: 4, unit: "hours" }, + { label: "Last 24 Hours", value: 24, unit: "hours" }, + { label: "Last 7 Days", value: 7, unit: "days" }, + ].map((option) => ( + + ))} +
+ +
+
+ )} +
+ + + + + + Refresh + +
+ + {isCustomDate && ( +
+
+ { + setStartTime(e.target.value); + setCurrentPage(1); + }} + className="px-3 py-2 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" + /> +
+ to +
+ { + setEndTime(e.target.value); + setCurrentPage(1); + }} + className="px-3 py-2 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" + /> +
+
+ )}
- {isCustomDate && ( -
-
- { - setStartTime(e.target.value); - setCurrentPage(1); - }} - className="px-3 py-2 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" - /> -
- to -
- { - setEndTime(e.target.value); - setCurrentPage(1); - }} - className="px-3 py-2 border rounded-md text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500" - /> -
-
- )} -
- -
- - Showing{" "} - {logs.isLoading - ? "..." - : logs.data - ? (currentPage - 1) * pageSize + 1 - : 0}{" "} - -{" "} - {logs.isLoading - ? "..." - : logs.data - ? Math.min(currentPage * pageSize, logs.data.total) - : 0}{" "} - of{" "} - {logs.isLoading - ? "..." - : logs.data - ? logs.data.total - : 0}{" "} - results - -
+
- Page {logs.isLoading ? "..." : currentPage} of{" "} + Showing{" "} {logs.isLoading ? "..." : logs.data - ? logs.data.total_pages - : 1} + ? (currentPage - 1) * pageSize + 1 + : 0}{" "} + -{" "} + {logs.isLoading + ? "..." + : logs.data + ? Math.min(currentPage * pageSize, logs.data.total) + : 0}{" "} + of{" "} + {logs.isLoading + ? "..." + : logs.data + ? logs.data.total + : 0}{" "} + results - - +
+ + Page {logs.isLoading ? "..." : currentPage} of{" "} + {logs.isLoading + ? "..." + : logs.data + ? logs.data.total_pages + : 1} + + + +
+ true} + onRowExpand={handleRowExpand} + expandedRequestId={expandedRequestId} + />
- true} - onRowExpand={handleRowExpand} - expandedRequestId={expandedRequestId} - /> -
- + )}
); } -function RequestViewer({ row }: { row: Row }) { +export function RequestViewer({ row }: { row: Row }) { // Helper function to clean metadata by removing specific fields - const getCleanedMetadata = (metadata: any) => { - const cleanedMetadata = {...metadata}; - if ('proxy_server_request' in cleanedMetadata) { - delete cleanedMetadata.proxy_server_request; - } - return cleanedMetadata; - }; - const formatData = (input: any) => { if (typeof input === "string") { try { @@ -655,8 +716,8 @@ function RequestViewer({ row }: { row: Row }) { // New helper function to get raw request const getRawRequest = () => { // First check if proxy_server_request exists in metadata - if (row.original.metadata?.proxy_server_request) { - return formatData(row.original.metadata.proxy_server_request); + if (row.original?.proxy_server_request) { + return formatData(row.original.proxy_server_request); } // Fall back to messages if proxy_server_request is empty return formatData(row.original.messages); @@ -849,8 +910,7 @@ function RequestViewer({ row }: { row: Row }) {

Metadata

-              {JSON.stringify(getCleanedMetadata(row.original.metadata), null, 2)}
+              {JSON.stringify(row.original.metadata, null, 2)}