From 5c0349a635292e0d893ab5f8a89c009e17ad7396 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 24 Apr 2026 22:29:15 -0700 Subject: [PATCH 1/2] [Feature] UI - Spend Logs: sortable Model and TTFT columns Extend the /spend/logs/ui sort_by whitelist to accept "model" and "ttft_ms", and wrap the Model and TTFT (s) column headers with the existing SortableHeader component so users can sort by either. TTFT has no stored column, so it is computed inline as (completionStartTime - startTime) milliseconds. Non-streaming rows (completionStartTime null or equal to endTime) yield NULL and the ORDER BY uses NULLS LAST so they always sort to the bottom regardless of direction, matching the existing "-" display in the UI. Adds parametrized backend tests for sort_by=model and a dedicated test covering streaming + non-streaming TTFT ordering in both directions. --- .../spend_management_endpoints.py | 27 ++- .../test_spend_management_endpoints.py | 224 ++++++++++++++++++ .../src/components/view_logs/columns.tsx | 26 +- 3 files changed, 267 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 805d0ec195..b1d3ee8e35 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1754,7 +1754,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 ), sort_by: str = fastapi.Query( default="startTime", - description="Sort logs by field: spend, total_tokens, startTime, or endTime", + description="Sort logs by field: spend, total_tokens, startTime, endTime, request_duration_ms, model, or ttft_ms", ), sort_order: Optional[str] = fastapi.Query( default="desc", @@ -1798,6 +1798,8 @@ async def ui_view_spend_logs( # noqa: PLR0915 "startTime", "endTime", "request_duration_ms", + "model", + "ttft_ms", } if sort_by not in valid_sort_fields: raise ProxyException( @@ -2045,13 +2047,22 @@ async def ui_view_spend_logs( # noqa: PLR0915 sql_params.append(f"%{error_message}%") p += 1 - # Quote column names that need quoting in SQL - _sql_col = ( - f'"{order_column}"' - if order_column in ("startTime", "endTime") - else order_column - ) + # Build the ORDER BY expression. NULLS LAST keeps rows without a + # meaningful value for the sorted column at the bottom regardless of + # direction. ttft_ms is computed from completionStartTime - startTime; + # non-streaming rows (where completionStartTime is null or equals + # endTime) yield NULL so they sort last. _sql_dir = "ASC" if order_direction == "asc" else "DESC" + if order_column == "ttft_ms": + _order_expr = ( + 'CASE WHEN "completionStartTime" IS NULL ' + 'OR "completionStartTime" = "endTime" THEN NULL ' + 'ELSE (EXTRACT(EPOCH FROM ("completionStartTime" - "startTime")) * 1000) END' + ) + elif order_column in ("startTime", "endTime"): + _order_expr = f'"{order_column}"' + else: + _order_expr = order_column sql_query = f""" SELECT @@ -2065,7 +2076,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms FROM "LiteLLM_SpendLogs" WHERE {" AND ".join(sql_conditions)} - ORDER BY {_sql_col} {_sql_dir} + ORDER BY {_order_expr} {_sql_dir} NULLS LAST LIMIT ${p} OFFSET ${p + 1} """ sql_params.extend([page_size, skip]) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 6f4e94d2c9..ab477151fa 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -783,6 +783,230 @@ async def test_ui_view_spend_logs_sort_by_request_duration_ms(client, monkeypatc app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sort_order,expected_request_ids", + [ + ("asc", ["req_anthropic", "req_gpt35", "req_gpt4"]), + ("desc", ["req_gpt4", "req_gpt35", "req_anthropic"]), + ], +) +async def test_ui_view_spend_logs_sort_by_model( + client, monkeypatch, sort_order, expected_request_ids +): + """Test that model is accepted as a valid sort_by field and orders alphabetically.""" + base_logs = [ + { + "request_id": "req_gpt4", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:00+00:00", + "endTime": "2025-01-01T00:00:01+00:00", + "model": "gpt-4", + }, + { + "request_id": "req_anthropic", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:01+00:00", + "endTime": "2025-01-01T00:00:02+00:00", + "model": "claude-3-opus", + }, + { + "request_id": "req_gpt35", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:02+00:00", + "endTime": "2025-01-01T00:00:03+00:00", + "model": "gpt-3.5-turbo", + }, + ] + + async def mock_count(*args, **kwargs): + return len(base_logs) + + async def mock_query_raw(sql_query, *params): + assert "model" in sql_query + assert "NULLS LAST" in sql_query + reverse = "DESC" in sql_query + sorted_logs = sorted( + base_logs, key=lambda x: x.get("model", ""), reverse=reverse + ) + page_size = params[-2] if len(params) >= 2 else 50 + skip = params[-1] if len(params) >= 1 else 0 + return sorted_logs[skip : skip + page_size] + + class MockPrismaClient: + def __init__(self): + self.db = MagicMock() + self.db.litellm_spendlogs = MagicMock() + self.db.litellm_spendlogs.count = AsyncMock(side_effect=mock_count) + self.db.query_raw = AsyncMock(side_effect=mock_query_raw) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MockPrismaClient()) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get( + "/spend/logs/ui", + params={ + "start_date": "2024-12-25 00:00:00", + "end_date": "2025-01-02 23:59:59", + "sort_by": "model", + "sort_order": sort_order, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200, response.text + data = response.json() + actual_ids = [log["request_id"] for log in data["data"]] + assert actual_ids == expected_request_ids + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_sort_by_ttft_ms(client, monkeypatch): + """sort_by=ttft_ms orders streaming rows by TTFT and pushes non-streaming rows last (NULLS LAST).""" + # req_fast_stream: TTFT = 100ms (streaming) + # req_slow_stream: TTFT = 2000ms (streaming) + # req_no_stream: completionStartTime == endTime (non-streaming, treated as NULL) + # req_null_stream: completionStartTime is null (non-streaming, NULL) + base_logs = [ + { + "request_id": "req_fast_stream", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:00+00:00", + "completionStartTime": "2025-01-01T00:00:00.100000+00:00", + "endTime": "2025-01-01T00:00:01+00:00", + "model": "gpt-4", + "_ttft_ms": 100, + }, + { + "request_id": "req_slow_stream", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:02+00:00", + "completionStartTime": "2025-01-01T00:00:04+00:00", + "endTime": "2025-01-01T00:00:05+00:00", + "model": "gpt-4", + "_ttft_ms": 2000, + }, + { + "request_id": "req_no_stream", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:06+00:00", + "completionStartTime": "2025-01-01T00:00:07+00:00", + "endTime": "2025-01-01T00:00:07+00:00", + "model": "gpt-4", + "_ttft_ms": None, + }, + { + "request_id": "req_null_stream", + "api_key": "sk-test-key", + "user": "user1", + "spend": 0.10, + "total_tokens": 100, + "startTime": "2025-01-01T00:00:08+00:00", + "completionStartTime": None, + "endTime": "2025-01-01T00:00:09+00:00", + "model": "gpt-4", + "_ttft_ms": None, + }, + ] + + async def mock_count(*args, **kwargs): + return len(base_logs) + + async def mock_query_raw(sql_query, *params): + # Endpoint must compute TTFT inline and use NULLS LAST. + assert "completionStartTime" in sql_query + assert "NULLS LAST" in sql_query + reverse = "DESC" in sql_query + non_null = [r for r in base_logs if r["_ttft_ms"] is not None] + nulls = [r for r in base_logs if r["_ttft_ms"] is None] + non_null.sort(key=lambda x: x["_ttft_ms"], reverse=reverse) + sorted_logs = non_null + nulls + page_size = params[-2] if len(params) >= 2 else 50 + skip = params[-1] if len(params) >= 1 else 0 + return [ + {k: v for k, v in row.items() if k != "_ttft_ms"} + for row in sorted_logs[skip : skip + page_size] + ] + + class MockPrismaClient: + def __init__(self): + self.db = MagicMock() + self.db.litellm_spendlogs = MagicMock() + self.db.litellm_spendlogs.count = AsyncMock(side_effect=mock_count) + self.db.query_raw = AsyncMock(side_effect=mock_query_raw) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MockPrismaClient()) + monkeypatch.setattr( + "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", + lambda user_api_key_dict: True, + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + # asc: fast stream, slow stream, then NULLs (non-streaming) last + response = client.get( + "/spend/logs/ui", + params={ + "start_date": "2024-12-25 00:00:00", + "end_date": "2025-01-02 23:59:59", + "sort_by": "ttft_ms", + "sort_order": "asc", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + actual_ids = [log["request_id"] for log in response.json()["data"]] + assert actual_ids[:2] == ["req_fast_stream", "req_slow_stream"] + assert set(actual_ids[2:]) == {"req_no_stream", "req_null_stream"} + + # desc: slow stream, fast stream, then NULLs still last + response = client.get( + "/spend/logs/ui", + params={ + "start_date": "2024-12-25 00:00:00", + "end_date": "2025-01-02 23:59:59", + "sort_by": "ttft_ms", + "sort_order": "desc", + }, + headers={"Authorization": "Bearer sk-test"}, + ) + assert response.status_code == 200, response.text + actual_ids = [log["request_id"] for log in response.json()["data"]] + assert actual_ids[:2] == ["req_slow_stream", "req_fast_stream"] + assert set(actual_ids[2:]) == {"req_no_stream", "req_null_stream"} + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_team_id(client, monkeypatch): mock_spend_logs = [ diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index a9110d31d3..27325217d4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -15,6 +15,8 @@ export const LOGS_SORT_FIELD_MAP = { spend: "spend", total_tokens: "total_tokens", request_duration_ms: "request_duration_ms", + model: "model", + ttft_ms: "ttft_ms", } as const; export type LogsSortField = keyof typeof LOGS_SORT_FIELD_MAP; @@ -272,7 +274,17 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] }, }, { - header: "TTFT (s)", + header: sortProps + ? () => ( + + ) + : "TTFT (s)", accessorKey: "completionStartTime", cell: (info: any) => { const row = info.row.original; @@ -328,7 +340,17 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] ), }, { - header: "Model", + header: sortProps + ? () => ( + + ) + : "Model", accessorKey: "model", cell: (info: any) => { const row = info.row.original; From 2047446546251abe4caed657e5e3ffc9da414c96 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 24 Apr 2026 22:52:04 -0700 Subject: [PATCH 2/2] Scope NULLS LAST to ttft_ms only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version appended NULLS LAST to every ORDER BY, which would silently change DESC semantics for any nullable sort column added to the whitelist later. Today the existing sort columns (spend, total_tokens, startTime, endTime, request_duration_ms, model) are all non-null in the result set, so the clause is a no-op for them — but the broader form is misleading. Apply NULLS LAST only when sorting by ttft_ms (the only column whose computed expression actually produces NULLs). Update the model test to assert the clause is absent for non-nullable columns. --- .../spend_tracking/spend_management_endpoints.py | 16 ++++++++++------ .../test_spend_management_endpoints.py | 5 ++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index b1d3ee8e35..4b8b341a4b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -2047,18 +2047,22 @@ async def ui_view_spend_logs( # noqa: PLR0915 sql_params.append(f"%{error_message}%") p += 1 - # Build the ORDER BY expression. NULLS LAST keeps rows without a - # meaningful value for the sorted column at the bottom regardless of - # direction. ttft_ms is computed from completionStartTime - startTime; - # non-streaming rows (where completionStartTime is null or equals - # endTime) yield NULL so they sort last. + # Build the ORDER BY expression. ttft_ms is computed from + # completionStartTime - startTime; non-streaming rows (where + # completionStartTime is null or equals endTime) yield NULL, so we + # append NULLS LAST in that case to keep them at the bottom regardless + # of direction. The other sort columns are non-null in the result set, + # so we leave the NULLS clause off and preserve their existing DESC + # semantics. _sql_dir = "ASC" if order_direction == "asc" else "DESC" + _nulls_clause = "" if order_column == "ttft_ms": _order_expr = ( 'CASE WHEN "completionStartTime" IS NULL ' 'OR "completionStartTime" = "endTime" THEN NULL ' 'ELSE (EXTRACT(EPOCH FROM ("completionStartTime" - "startTime")) * 1000) END' ) + _nulls_clause = " NULLS LAST" elif order_column in ("startTime", "endTime"): _order_expr = f'"{order_column}"' else: @@ -2076,7 +2080,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms FROM "LiteLLM_SpendLogs" WHERE {" AND ".join(sql_conditions)} - ORDER BY {_order_expr} {_sql_dir} NULLS LAST + ORDER BY {_order_expr} {_sql_dir}{_nulls_clause} LIMIT ${p} OFFSET ${p + 1} """ sql_params.extend([page_size, skip]) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index ab477151fa..4bcabfe853 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -833,7 +833,10 @@ async def test_ui_view_spend_logs_sort_by_model( async def mock_query_raw(sql_query, *params): assert "model" in sql_query - assert "NULLS LAST" in sql_query + # model is non-nullable in the schema, so NULLS LAST should NOT be + # appended — only ttft_ms gets that clause. This guards against + # accidentally widening the change to all sort columns. + assert "NULLS LAST" not in sql_query reverse = "DESC" in sql_query sorted_logs = sorted( base_logs, key=lambda x: x.get("model", ""), reverse=reverse