From ed4b654934d135a8f923195055ac796f8686390e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 11:27:09 -0800 Subject: [PATCH 01/13] [Fix] Include all SpendLogsMetadata keys in spend logs payload The dict comprehension in _get_spend_logs_metadata was only including metadata keys that existed in the input dict. After user_api_key_project_id was added to SpendLogsMetadata, payloads missing that key in input would not include it in output, causing test_spend_logs_payload to fail. Use metadata.get(key) instead of filtering with `if key in metadata` to ensure all SpendLogsMetadata keys are always present (defaulting to None), consistent with the metadata-is-None branch. Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/proxy/spend_tracking/spend_tracking_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 0796fdcc0b..d517c76a08 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -98,9 +98,8 @@ def _get_spend_logs_metadata( # Filter the metadata dictionary to include only the specified keys clean_metadata = SpendLogsMetadata( **{ # type: ignore - key: metadata[key] + key: metadata.get(key) for key in SpendLogsMetadata.__annotations__.keys() - if key in metadata } ) clean_metadata["applied_guardrails"] = applied_guardrails From e5619c39a0b8e2cfbf44ea38b3f82aecd7811087 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 11:33:20 -0800 Subject: [PATCH 02/13] [Fix] Update UI tests for GuardrailViewer rewrite and AllModelsTab QueryClient GuardrailViewer was rewritten from ant-design Collapse to a custom card layout. Tests now match the new component: updated header text, ms-based duration, expand-to-reveal provider details, and removed ant-collapse references. AllModelsTab tests failed because ModelSettingsModal now uses useMutation via useStoreModelInDB. Switched from bare render() to renderWithProviders() which wraps in QueryClientProvider. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../components/AllModelsTab.test.tsx | 18 ++-- .../GuardrailViewer/GuardrailViewer.test.tsx | 99 +++++++++++-------- 2 files changed, 65 insertions(+), 52 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx index 813a365d36..34c1c3ca4b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx @@ -1,5 +1,5 @@ import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized"; -import { render, screen, waitFor } from "@testing-library/react"; +import { renderWithProviders, screen, waitFor } from "../../../../../tests/test-utils"; import { beforeEach, describe, expect, it, vi } from "vitest"; import AllModelsTab from "./AllModelsTab"; @@ -116,7 +116,7 @@ describe("AllModelsTab", () => { mockUseModelCostMap.mockReturnValueOnce(createModelCostMapMock({})); - render(); + renderWithProviders(); expect(screen.getByText("Current Team:")).toBeInTheDocument(); }); @@ -172,7 +172,7 @@ describe("AllModelsTab", () => { mockUseModelsInfo.mockReturnValue({ data: modelData, isLoading: false, error: null }); - render(); + renderWithProviders(); // Component shows API total_count (2), not filtered count // Since default is "personal" team and models don't have direct_access, they're filtered out @@ -233,7 +233,7 @@ describe("AllModelsTab", () => { mockUseModelsInfo.mockReturnValue({ data: modelData, isLoading: false, error: null }); - render(); + renderWithProviders(); // Component shows API total_count (2), not filtered count // Since default is "personal" team and models don't have direct_access, they're filtered out @@ -280,7 +280,7 @@ describe("AllModelsTab", () => { mockUseModelsInfo.mockReturnValue({ data: modelData, isLoading: false, error: null }); - render(); + renderWithProviders(); // Component shows API total_count (2), but only 1 model has direct_access await waitFor(() => { @@ -338,7 +338,7 @@ describe("AllModelsTab", () => { mockUseModelsInfo.mockReturnValue({ data: modelData, isLoading: false, error: null }); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("Config Model")).toBeInTheDocument(); @@ -380,7 +380,7 @@ describe("AllModelsTab", () => { mockUseModelsInfo.mockReturnValue({ data: modelData, isLoading: false, error: null }); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("Defined in config")).toBeInTheDocument(); @@ -426,7 +426,7 @@ describe("AllModelsTab", () => { return { data: page1Data, isLoading: false, error: null }; }); - render(); + renderWithProviders(); await waitFor(() => { // Component calculates: ((1-1)*50)+1 = 1, Math.min(1*50, 2) = 2 @@ -479,7 +479,7 @@ describe("AllModelsTab", () => { return { data: singlePageData, isLoading: false, error: null }; }); - render(); + renderWithProviders(); await waitFor(() => { expect(screen.getByText("Showing 1 - 1 of 1 results")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 95120f6057..28991ebfa0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -19,68 +19,62 @@ describe("GuardrailViewer", () => { vi.resetModules(); }); - it("shows header, status pill color, duration rounding, and time labels", () => { + it("shows header, status pill, and duration", () => { const data = makeGuardrailInformation({ duration: 1.23456, guardrail_status: "success" }); renderWithProviders(); - expect(screen.getByText("Guardrail Information")).toBeInTheDocument(); - // header status pill (success => green) - const statusBadges = screen.getAllByText("success"); - // there are two status locations: header chip and grid "Status" - expect(statusBadges.length).toBeGreaterThanOrEqual(1); - // Quick class assertion for at least one of them - expect(statusBadges[0].className).toMatch(/bg-green-100/); + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + // header shows passed count + expect(screen.getByText(/1 Passed/)).toBeInTheDocument(); + // The PASSED badge in the evaluation card + expect(screen.getByText("PASSED")).toBeInTheDocument(); - // duration displays with 4 decimals - expect(screen.getByText(/1\.2346s/)).toBeInTheDocument(); - - // time labels exist - expect(screen.getByText("Start Time:")).toBeInTheDocument(); - expect(screen.getByText("End Time:")).toBeInTheDocument(); + // duration displays in ms format: Math.round(1.23456 * 1000) = 1235 + expect(screen.getByText("1235ms")).toBeInTheDocument(); }); - it("calculates and displays masked entity totals with pluralization", () => { + it("calculates and displays masked entity totals", async () => { + const user = userEvent.setup(); const data = makeGuardrailInformation({ masked_entity_count: { EMAIL_ADDRESS: 2, PHONE_NUMBER: 1 }, }); renderWithProviders(); - expect(screen.getByText("3 masked entities")).toBeInTheDocument(); - // summary chips for each entry + // In collapsed state, the match count badge is visible + expect(screen.getByText("3 matched")).toBeInTheDocument(); + + // Expand the evaluation card to see entity details + await user.click(screen.getByText("pii-rail")); + // summary chips for each entry inside expanded card expect(screen.getByText("EMAIL_ADDRESS: 2")).toBeInTheDocument(); expect(screen.getByText("PHONE_NUMBER: 1")).toBeInTheDocument(); }); - it("hides masked badge & summary when count is zero/empty", () => { + it("hides matched badge when count is zero/empty", () => { const data = makeGuardrailInformation({ masked_entity_count: {} }); renderWithProviders(); - expect(screen.queryByText(/masked entity/)).not.toBeInTheDocument(); - expect(screen.queryByText("Masked Entity Summary")).not.toBeInTheDocument(); + expect(screen.queryByText(/matched/)).not.toBeInTheDocument(); }); - it("toggles main section open/closed and chevron rotation class", async () => { + it("toggles evaluation card open/closed on click", async () => { const user = userEvent.setup(); - const data = makeGuardrailInformation(); - const { container } = renderWithProviders(); - - const header = screen.getByText("Guardrail Information").closest(".ant-collapse-header")!; - // Initially expanded (content is visible) - expect(screen.getByText("Masked Entity Summary")).toBeInTheDocument(); - - // Click to collapse - await user.click(header); - // Wait for collapse animation and content to be hidden - await waitFor(() => { - const contentBox = container.querySelector(".ant-collapse-content-box"); - expect(contentBox).not.toBeVisible(); + const data = makeGuardrailInformation({ + masked_entity_count: { EMAIL_ADDRESS: 2 }, }); + renderWithProviders(); - // Click to expand again - await user.click(header); - // Wait for expand animation + // Initially collapsed — masked entity details not visible + expect(screen.queryByText("EMAIL_ADDRESS: 2")).not.toBeInTheDocument(); + + // Click to expand + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("EMAIL_ADDRESS: 2")).toBeInTheDocument(); + + // Click again to collapse + await user.click(screen.getByText("pii-rail")); await waitFor(() => { - expect(screen.getByText("Masked Entity Summary")).toBeVisible(); + expect(screen.queryByText("EMAIL_ADDRESS: 2")).not.toBeInTheDocument(); }); }); @@ -97,6 +91,9 @@ describe("GuardrailViewer", () => { }); renderWithProviders(); + // Expand the card to see provider-specific content + const user = userEvent.setup(); + await user.click(screen.getByText("pii-rail")); expect(screen.getByTestId("presidio-mock")).toHaveTextContent("presidio 2"); }); @@ -112,6 +109,10 @@ describe("GuardrailViewer", () => { guardrail_response: [makeEntity()], }); renderWithProviders(); + + // Expand the card to see provider-specific content + const user = userEvent.setup(); + await user.click(screen.getByText("pii-rail")); expect(screen.getByTestId("presidio-mock")).toHaveTextContent("count:1"); }); @@ -127,22 +128,31 @@ describe("GuardrailViewer", () => { guardrail_response: makeBedrockResponse({ action: "GUARDRAIL_INTERVENED" }), }); renderWithProviders(); + + // Expand the card to see provider-specific content + const user = userEvent.setup(); + await user.click(screen.getByText("pii-rail")); expect(screen.getByTestId("bedrock-mock")).toHaveTextContent("GUARDRAIL_INTERVENED"); }); - it("unknown provider renders neither Presidio nor Bedrock details", () => { + it("unknown provider renders neither Presidio nor Bedrock details", async () => { + const user = userEvent.setup(); const data = makeGuardrailInformation({ guardrail_provider: "unknown", }); renderWithProviders(); - // Summary still present - expect(screen.getByText("Guardrail Information")).toBeInTheDocument(); - // No provider sections + // Header still present + expect(screen.getByText("Guardrails & Policy Compliance")).toBeInTheDocument(); + + // Expand the card + await user.click(screen.getByText("pii-rail")); + // No Presidio or Bedrock sections expect(screen.queryByText(/Detected Entities/)).not.toBeInTheDocument(); expect(screen.queryByText(/Raw Bedrock Guardrail Response/)).not.toBeInTheDocument(); }); - it("integration: renders with real Bedrock details without mocks", () => { + it("integration: renders with real Bedrock details without mocks", async () => { + const user = userEvent.setup(); const data = makeGuardrailInformation({ guardrail_provider: "bedrock", guardrail_response: makeBedrockResponse({ @@ -152,6 +162,9 @@ describe("GuardrailViewer", () => { }); renderWithProviders(); + // Expand the card to reveal Bedrock details + await user.click(screen.getByText("pii-rail")); + // Bedrock summary bits expect(screen.getByText("Outputs")).toBeInTheDocument(); expect(screen.getByText("ok")).toBeInTheDocument(); From c27b65d09e15316edd9a2705e45f0cd61047d8b3 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 14:03:01 -0800 Subject: [PATCH 03/13] [Fix] Replace deprecated claude-3-7-sonnet-20250219 with claude-sonnet-4-5-20250929 in test_completion Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/local_testing/test_completion.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index b9a366d9f3..51ed6a53bb 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -286,7 +286,7 @@ def test_completion_claude_3_empty_response(): }, ] try: - response = litellm.completion(model="claude-3-7-sonnet-20250219", messages=messages) + response = litellm.completion(model="claude-sonnet-4-5-20250929", messages=messages) print(response) except litellm.InternalServerError as e: pytest.skip(f"InternalServerError - {str(e)}") @@ -313,7 +313,7 @@ def test_completion_claude_3(): try: # test without max tokens response = completion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=messages, ) # Add any assertions, here to check response args @@ -326,7 +326,7 @@ def test_completion_claude_3(): @pytest.mark.parametrize( "model", - ["anthropic/claude-3-7-sonnet-20250219", "anthropic.claude-3-sonnet-20240229-v1:0"], + ["anthropic/claude-sonnet-4-5-20250929", "anthropic.claude-3-sonnet-20240229-v1:0"], ) def test_completion_claude_3_function_call(model): litellm.set_verbose = True @@ -411,7 +411,7 @@ def test_completion_claude_3_function_call(model): "model, api_key, api_base", [ ("gpt-3.5-turbo", None, None), - ("claude-3-7-sonnet-20250219", None, None), + ("claude-sonnet-4-5-20250929", None, None), ("anthropic.claude-3-sonnet-20240229-v1:0", None, None), # ( # "azure_ai/command-r-plus", @@ -512,7 +512,7 @@ async def test_anthropic_no_content_error(): try: litellm.drop_params = True response = await litellm.acompletion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", api_key=os.getenv("ANTHROPIC_API_KEY"), messages=[ { @@ -630,7 +630,7 @@ def test_completion_claude_3_multi_turn_conversations(): ] try: response = completion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=messages, ) print(response) @@ -644,7 +644,7 @@ def test_completion_claude_3_stream(): try: # test without max tokens response = completion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=messages, max_tokens=10, stream=True, @@ -669,7 +669,7 @@ def encode_image(image_path): [ "gpt-4o", "azure/gpt-4.1-mini", - "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-sonnet-4-5-20250929", ], ) # def test_completion_base64(model): From 5354cb26e1008520fbb18e59d34de6e6641a0b46 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 14:05:23 -0800 Subject: [PATCH 04/13] [Fix] Replace deprecated claude-3-7-sonnet in test_anthropic_completion, add store to OPENAI_CHAT_COMPLETION_PARAMS Replace claude-3-7-sonnet-20250219 with claude-sonnet-4-5-20250929 in test_anthropic_completion.py (9 instances). Add missing "store" param to OPENAI_CHAT_COMPLETION_PARAMS to fix test_store_in_openai_chat_completion_params. Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/constants.py | 1 + .../test_anthropic_completion.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index ee1b69f145..89992b459c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -603,6 +603,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [ "prompt_cache_retention", "safety_identifier", "verbosity", + "store", ] OPENAI_TRANSCRIPTION_PARAMS = [ diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 405e0d2c82..e240437178 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1039,8 +1039,8 @@ def test_anthropic_citations_api_streaming(): @pytest.mark.parametrize( "model", [ - "anthropic/claude-3-7-sonnet-20250219", - "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic/claude-sonnet-4-5-20250929", + "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) def test_anthropic_thinking_output(model): @@ -1068,9 +1068,9 @@ def test_anthropic_thinking_output(model): @pytest.mark.parametrize( "model", [ - "anthropic/claude-3-7-sonnet-20250219", - # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", - # "bedrock/invoke/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic/claude-sonnet-4-5-20250929", + # "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + # "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) def test_anthropic_thinking_output_stream(model): @@ -1152,8 +1152,8 @@ def test_anthropic_custom_headers(): @pytest.mark.parametrize( "model", [ - "anthropic/claude-3-7-sonnet-20250219", - # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic/claude-sonnet-4-5-20250929", + # "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) def test_anthropic_thinking_in_assistant_message(model): @@ -1189,8 +1189,8 @@ def test_anthropic_thinking_in_assistant_message(model): @pytest.mark.parametrize( "model", [ - "anthropic/claude-3-7-sonnet-20250219", - # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "anthropic/claude-sonnet-4-5-20250929", + # "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", ], ) def test_anthropic_redacted_thinking_in_assistant_message(model): @@ -1226,7 +1226,7 @@ def test_just_system_message(): litellm._turn_on_debug() litellm.modify_params = True params = { - "model": "anthropic/claude-3-7-sonnet-20250219", + "model": "anthropic/claude-sonnet-4-5-20250929", "messages": [{"role": "system", "content": "You are a helpful assistant."}], } From e6b9bef949ee50cbc7849b6a8105238c0cfc7b35 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 14:16:07 -0800 Subject: [PATCH 05/13] [Fix] Fix flaky tests: spend logs metadata keys, proxy CLI isolation, Redis TTL uniqueness - Add new SpendLogsMetadata keys to ignored_keys in spend logs tests (regression from ccecc10c82 which intentionally includes all keys) - Mock PrismaManager.setup_database and should_update_prisma_schema in proxy CLI tests to prevent real DB migrations from running in CI - Use CliRunner(mix_stderr=False) to fix Click stream lifecycle issues - Use unique UUID suffix for Redis TTL test keys to avoid stale state Co-Authored-By: Claude Opus 4.6 (1M context) --- .../hooks/test_parallel_request_limiter_v3.py | 7 +++-- .../test_spend_management_endpoints.py | 12 ++++++++ tests/test_litellm/proxy/test_proxy_cli.py | 28 +++++++++---------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 134fc84965..02d51cc4a8 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -1176,8 +1176,11 @@ async def test_async_increment_tokens_with_ttl_preservation(): ) # Test keys - use hash tags to ensure they map to same Redis cluster slot - test_key_with_ttl = "{test_ttl}:with_ttl" - test_key_without_ttl = "{test_ttl}:without_ttl" + # Use a unique suffix per test run to avoid stale state from prior runs + import uuid + unique_suffix = str(uuid.uuid4())[:8] + test_key_with_ttl = f"{{test_ttl}}:with_ttl:{unique_suffix}" + test_key_without_ttl = f"{{test_ttl}}:without_ttl:{unique_suffix}" try: # Clean up any existing test keys diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index eabaec8c20..f275681bde 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -287,6 +287,18 @@ ignored_keys = [ "metadata.additional_usage_values.speed", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", + "metadata.user_api_key", + "metadata.user_api_key_alias", + "metadata.user_api_key_team_id", + "metadata.user_api_key_project_id", + "metadata.user_api_key_org_id", + "metadata.user_api_key_user_id", + "metadata.user_api_key_team_alias", + "metadata.spend_logs_metadata", + "metadata.requester_ip_address", + "metadata.status", + "metadata.proxy_server_request", + "metadata.error_information", ] MODEL_LIST = [ diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 543547943d..b3cf830ab0 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -225,13 +225,15 @@ class TestProxyInitializationHelpers: assert modified_url == "" @patch("uvicorn.run") - @patch("atexit.register") # 🔥 critical - def test_skip_server_startup(self, mock_atexit_register, mock_uvicorn_run): + @patch("atexit.register") # critical + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch("litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False) + def test_skip_server_startup(self, mock_should_update, mock_setup_db, mock_atexit_register, mock_uvicorn_run): from click.testing import CliRunner from litellm.proxy.proxy_cli import run_server - runner = CliRunner() + runner = CliRunner(mix_stderr=False) mock_proxy_module = MagicMock( app=MagicMock(), @@ -594,7 +596,7 @@ class TestHealthAppFactory: from litellm.proxy.proxy_cli import run_server - runner = CliRunner() + runner = CliRunner(mix_stderr=False) # Mock subprocess.run to simulate prisma being available mock_subprocess_run.return_value = MagicMock(returncode=0) @@ -602,20 +604,18 @@ class TestHealthAppFactory: # Mock should_update_prisma_schema to return True (so setup_database gets called) mock_should_update_schema.return_value = True - mock_app = MagicMock() - mock_proxy_config = MagicMock() - mock_key_mgmt = MagicMock() - mock_save_worker_config = MagicMock() + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) with patch.dict( "sys.modules", { - "proxy_server": MagicMock( - app=mock_app, - ProxyConfig=mock_proxy_config, - KeyManagementSettings=mock_key_mgmt, - save_worker_config=mock_save_worker_config, - ) + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, }, ), patch( "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" From 65dc7556a8c78440bb2dea76248a9902e6b3e1ac Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 19 Feb 2026 14:22:47 -0800 Subject: [PATCH 06/13] [Fix] Fix web search model info regression, deprecated prompt caching model, undocumented env keys - Revert test_anthropic_web_search_in_model_info to use claude-3-5-haiku-latest (model info test doesn't make API calls, so the -latest alias is fine here) - Replace claude-3-7-sonnet-20250219 with claude-sonnet-4-5-20250929 in test_anthropic_prompt_caching.py (10 instances) - Include pending doc updates for COMPETITOR_LLM_TEMPERATURE and MAX_COMPETITOR_NAMES env vars in config_settings.md Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/my-website/docs/proxy/config_settings.md | 2 ++ .../test_anthropic_prompt_caching.py | 20 +++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 9e3b5e9097..5b255f0188 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -485,6 +485,7 @@ router_settings: | CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache | CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service | COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com +| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3 | DATABASE_HOST | Hostname for the database server | DATABASE_NAME | Name of the database | DATABASE_PASSWORD | Password for the database user @@ -806,6 +807,7 @@ router_settings: | LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000 | LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0 | LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50% +| MAX_COMPETITOR_NAMES | Maximum number of competitor names allowed in policy template enrichment. Default is 100 | MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000 | MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200 | MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0 diff --git a/tests/local_testing/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py index c8589dd884..417a7335a8 100644 --- a/tests/local_testing/test_anthropic_prompt_caching.py +++ b/tests/local_testing/test_anthropic_prompt_caching.py @@ -57,7 +57,7 @@ async def test_litellm_anthropic_prompt_caching_tools(): "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello!"}], - "model": "claude-3-7-sonnet-20250219", + "model": "claude-sonnet-4-5-20250929", "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 12, "output_tokens": 6}, @@ -74,7 +74,7 @@ async def test_litellm_anthropic_prompt_caching_tools(): # Act: Call the litellm.acompletion function response = await litellm.acompletion( api_key="mock_api_key", - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ {"role": "user", "content": "What's the weather like in Boston today?"} ], @@ -154,7 +154,7 @@ async def test_litellm_anthropic_prompt_caching_tools(): } ], "max_tokens": 64000, - "model": "claude-3-7-sonnet-20250219", + "model": "claude-sonnet-4-5-20250929", } mock_post.assert_called_once_with( @@ -240,7 +240,7 @@ async def test_anthropic_vertex_ai_prompt_caching(anthropic_messages, sync_mode) async def test_anthropic_api_prompt_caching_basic(): litellm.set_verbose = True response = await litellm.acompletion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ # System Message { @@ -308,7 +308,7 @@ async def test_anthropic_api_prompt_caching_basic_with_cache_creation(): litellm.set_verbose = True response = await litellm.acompletion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ # System Message { @@ -460,7 +460,7 @@ async def test_anthropic_api_prompt_caching_with_content_str(): async def test_anthropic_api_prompt_caching_no_headers(): litellm.set_verbose = True response = await litellm.acompletion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ # System Message { @@ -520,7 +520,7 @@ async def test_anthropic_api_prompt_caching_no_headers(): @pytest.mark.asyncio() async def test_anthropic_api_prompt_caching_streaming(): response = await litellm.acompletion( - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ # System Message { @@ -603,7 +603,7 @@ async def test_litellm_anthropic_prompt_caching_system(): "type": "message", "role": "assistant", "content": [{"type": "text", "text": "Hello!"}], - "model": "claude-3-7-sonnet-20250219", + "model": "claude-sonnet-4-5-20250929", "stop_reason": "end_turn", "stop_sequence": None, "usage": {"input_tokens": 12, "output_tokens": 6}, @@ -620,7 +620,7 @@ async def test_litellm_anthropic_prompt_caching_system(): # Act: Call the litellm.acompletion function response = await litellm.acompletion( api_key="mock_api_key", - model="anthropic/claude-3-7-sonnet-20250219", + model="anthropic/claude-sonnet-4-5-20250929", messages=[ { "role": "system", @@ -681,7 +681,7 @@ async def test_litellm_anthropic_prompt_caching_system(): } ], "max_tokens": 64000, - "model": "claude-3-7-sonnet-20250219", + "model": "claude-sonnet-4-5-20250929", } mock_post.assert_called_once_with( From 36fd14357ceedb3ca9fbefc8dc5ca16c1d1abf89 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 08:46:45 +0530 Subject: [PATCH 07/13] FIx: replace deprecated claude-3-7-sonnet-20250219 with claude-4-sonnet-20250514 --- .../anthropic/test_anthropic_reasoning_effort.py | 8 ++++---- .../test_convert_dict_to_chat_completion.py | 2 +- .../test_amazing_vertex_completion.py | 2 +- tests/local_testing/test_function_calling.py | 2 +- tests/local_testing/test_streaming.py | 4 ++-- .../base_anthropic_messages_test.py | 4 ++-- .../test_anthropic_passthrough.py | 2 +- .../test_spend_management_endpoints.py | 14 +++++++------- .../test_session_handler.py | 4 ++-- tests/test_litellm/test_utils.py | 8 ++++---- 10 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py index 89da8d87e6..98ae7148c7 100644 --- a/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py +++ b/tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py @@ -19,7 +19,7 @@ class TestMapReasoningEffort: def test_none_returns_none_for_other_models(self): """reasoning_effort=None should return None for non-Opus models.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort=None, model="claude-3-7-sonnet-20250219" + reasoning_effort=None, model="claude-4-sonnet-20250514" ) assert result is None @@ -37,14 +37,14 @@ class TestMapReasoningEffort: def test_other_model_low_returns_enabled_with_budget(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="low", model="claude-3-7-sonnet-20250219" + reasoning_effort="low", model="claude-4-sonnet-20250514" ) assert result["type"] == "enabled" assert "budget_tokens" in result def test_other_model_high_returns_enabled_with_budget(self): result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="high", model="claude-3-7-sonnet-20250219" + reasoning_effort="high", model="claude-4-sonnet-20250514" ) assert result["type"] == "enabled" assert "budget_tokens" in result @@ -59,6 +59,6 @@ class TestMapReasoningEffort: def test_none_string_returns_none_for_other_models(self): """reasoning_effort='none' should return None for non-Opus models.""" result = AnthropicConfig._map_reasoning_effort( - reasoning_effort="none", model="claude-3-7-sonnet-20250219" + reasoning_effort="none", model="claude-4-sonnet-20250514" ) assert result is None diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 3b2087d25e..5a37a5ec93 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -864,7 +864,7 @@ def test_convert_to_model_response_object_with_thinking_content(): "response_object": { "id": "chatcmpl-8cc87354-70f3-4a14-b71b-332e965d98d2", "created": 1741057687, - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "object": "chat.completion", "system_fingerprint": None, "choices": [ diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index 0078483c73..5daabf083e 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -3242,7 +3242,7 @@ def vertex_ai_anthropic_thinking_mock_response(*args, **kwargs): "id": "msg_vrtx_011pL6Np3MKxXL3R8theMRJW", "type": "message", "role": "assistant", - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "content": [ { "type": "thinking", diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index e2f9c6d834..e47b32a01f 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -158,7 +158,7 @@ def test_aaparallel_function_call(model): @pytest.mark.parametrize( "model", [ - "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-4-sonnet-20250514", "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", ], ) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index ee208b5e0e..b3f13e8a4b 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -1387,7 +1387,7 @@ def test_bedrock_claude_3_streaming(): @pytest.mark.parametrize( "model", [ - "claude-3-7-sonnet-20250219", + "claude-4-sonnet-20250514", "cohere.command-r-plus-v1:0", # bedrock "gpt-3.5-turbo", ], @@ -2883,7 +2883,7 @@ def test_completion_claude_3_function_call_with_streaming(): try: # test without max tokens response = completion( - model="claude-3-7-sonnet-20250219", + model="claude-4-sonnet-20250514", messages=messages, tools=tools, tool_choice="required", diff --git a/tests/pass_through_tests/base_anthropic_messages_test.py b/tests/pass_through_tests/base_anthropic_messages_test.py index 90d00ccb1a..e86e58de33 100644 --- a/tests/pass_through_tests/base_anthropic_messages_test.py +++ b/tests/pass_through_tests/base_anthropic_messages_test.py @@ -54,7 +54,7 @@ class BaseAnthropicMessagesTest(ABC): print("making request to anthropic passthrough with thinking") client = self.get_client() response = client.messages.create( - model="claude-3-7-sonnet-20250219", + model="claude-4-sonnet-20250514", max_tokens=20000, thinking={"type": "enabled", "budget_tokens": 16000}, messages=[ @@ -75,7 +75,7 @@ class BaseAnthropicMessagesTest(ABC): collected_response = [] client = self.get_client() with client.messages.stream( - model="claude-3-7-sonnet-20250219", + model="claude-4-sonnet-20250514", max_tokens=20000, thinking={"type": "enabled", "budget_tokens": 16000}, messages=[ diff --git a/tests/pass_through_tests/test_anthropic_passthrough.py b/tests/pass_through_tests/test_anthropic_passthrough.py index 1a2d1b28ab..81bbb88952 100644 --- a/tests/pass_through_tests/test_anthropic_passthrough.py +++ b/tests/pass_through_tests/test_anthropic_passthrough.py @@ -313,7 +313,7 @@ async def test_anthropic_messages_streaming_cost_injection(): } payload = { - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "max_tokens": 10, "stream": True, "messages": [{"role": "user", "content": "Say 'Hi'"}], diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index f275681bde..fcfc696f00 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1275,7 +1275,7 @@ class TestSpendLogsPayload: mock_response.json.return_value = { "content": [{"text": "Hi! My name is Claude.", "type": "text"}], "id": "msg_013Zva2CMHLNnXjNJJKqJ2EF", - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "role": "assistant", "stop_reason": "end_turn", "stop_sequence": None, @@ -1302,7 +1302,7 @@ class TestSpendLogsPayload: client, "post", side_effect=self.mock_anthropic_response ): response = await litellm.acompletion( - model="claude-3-7-sonnet-20250219", + model="claude-4-sonnet-20250514", messages=[{"role": "user", "content": "Hello, world!"}], metadata={"user_api_key_end_user_id": "test_user_1"}, client=client, @@ -1331,10 +1331,10 @@ class TestSpendLogsPayload: "completionStartTime": datetime.datetime( 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc ), - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -1376,7 +1376,7 @@ class TestSpendLogsPayload: { "model_name": "my-anthropic-model-group", "litellm_params": { - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", }, "model_info": { "id": "my-unique-model-id", @@ -1423,10 +1423,10 @@ class TestSpendLogsPayload: "completionStartTime": datetime.datetime( 2025, 3, 24, 22, 2, 42, 989132, tzinfo=datetime.timezone.utc ), - "model": "claude-3-7-sonnet-20250219", + "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-3-7-sonnet-20250219", "model_map_value": {"key": "claude-3-7-sonnet-20250219", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "guardrail_information": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py index b0a232a7bf..9279ce2611 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py @@ -319,7 +319,7 @@ async def test_should_check_cold_storage_for_full_payload(): ] } ], - "model": "anthropic/claude-3-7-sonnet-20250219", + "model": "anthropic/claude-4-sonnet-20250514", "stream": True, "litellm_trace_id": "16b86861-c120-4ecb-865b-4d2238bfd8f0" } @@ -333,7 +333,7 @@ async def test_should_check_cold_storage_for_full_payload(): "content": "Hello, this is a regular message" } ], - "model": "anthropic/claude-3-7-sonnet-20250219", + "model": "anthropic/claude-4-sonnet-20250514", "stream": True } diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 5bfb3bd879..31f492a45b 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -429,7 +429,7 @@ def test_anthropic_web_search_in_model_info(): litellm.model_cost = litellm.get_model_cost_map(url="") supported_models = [ - "anthropic/claude-3-7-sonnet-20250219", + "anthropic/claude-4-sonnet-20250514", "anthropic/claude-sonnet-4-5-20250929", "anthropic/claude-3-5-sonnet-20241022", "anthropic/claude-3-5-haiku-20241022", @@ -1050,7 +1050,7 @@ def test_supports_computer_use_utility(): try: # Test a model known to support computer_use from backup JSON supports_cu_anthropic = supports_computer_use( - model="anthropic/claude-3-7-sonnet-20250219" + model="anthropic/claude-4-sonnet-20250514" ) assert supports_cu_anthropic is True @@ -1073,7 +1073,7 @@ def test_supports_computer_use_utility(): def test_get_model_info_shows_supports_computer_use(): """ Tests if 'supports_computer_use' is correctly retrieved by get_model_info. - We'll use 'claude-3-7-sonnet-20250219' as it's configured + We'll use 'claude-4-sonnet-20250514' as it's configured in the backup JSON to have supports_computer_use: True. """ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" @@ -1082,7 +1082,7 @@ def test_get_model_info_shows_supports_computer_use(): litellm.model_cost = litellm.get_model_cost_map(url="") # This model should have 'supports_computer_use': True in the backup JSON - model_known_to_support_computer_use = "claude-3-7-sonnet-20250219" + model_known_to_support_computer_use = "claude-4-sonnet-20250514" info = litellm.get_model_info(model_known_to_support_computer_use) print(f"Info for {model_known_to_support_computer_use}: {info}") From 61eaf960461c659c9f74cdffdb4da0f7d4a2d841 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 08:53:16 +0530 Subject: [PATCH 08/13] Fix passthrough tests --- .../test_passthrough_registry_updates.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/pass_through_unit_tests/test_passthrough_registry_updates.py b/tests/pass_through_unit_tests/test_passthrough_registry_updates.py index 125ffdb6fa..87309ed36e 100644 --- a/tests/pass_through_unit_tests/test_passthrough_registry_updates.py +++ b/tests/pass_through_unit_tests/test_passthrough_registry_updates.py @@ -18,7 +18,9 @@ def test_update_pass_through_route_updates_registry(): # Setup - Unique IDs to avoid collision with other tests endpoint_id = "regression-test-endpoint" path = "/regression-test-path" - route_key = f"{endpoint_id}:exact:{path}" + # Default methods are sorted: DELETE,GET,PATCH,POST,PUT + methods_str = "DELETE,GET,PATCH,POST,PUT" + route_key = f"{endpoint_id}:exact:{path}:{methods_str}" target = "http://example.com" # Cleanup: Ensure clean state before test @@ -90,7 +92,9 @@ def test_update_subpath_route_updates_registry(): # Setup endpoint_id = "regression-test-subpath" path = "/regression-test-wildcard" - route_key = f"{endpoint_id}:subpath:{path}" + # Default methods are sorted: DELETE,GET,PATCH,POST,PUT + methods_str = "DELETE,GET,PATCH,POST,PUT" + route_key = f"{endpoint_id}:subpath:{path}:{methods_str}" target = "http://example.com" if route_key in _registered_pass_through_routes: From adb91d442ab8cc4eb87cef2c5c9a3f64d52ddbd9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 09:01:29 +0530 Subject: [PATCH 09/13] Fix: test_pass_through_endpoint_bing --- .../test_pass_through_endpoints.py | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/local_testing/test_pass_through_endpoints.py b/tests/local_testing/test_pass_through_endpoints.py index 2795bc918b..3ca504fc6e 100644 --- a/tests/local_testing/test_pass_through_endpoints.py +++ b/tests/local_testing/test_pass_through_endpoints.py @@ -3,6 +3,7 @@ import sys from litellm._uuid import uuid from functools import partial from typing import Optional +from urllib.parse import urlparse, parse_qs import pytest from fastapi import FastAPI @@ -535,10 +536,27 @@ async def test_pass_through_endpoint_bing(client, monkeypatch): first_transformed_url = captured_requests[0][1]["url"] second_transformed_url = captured_requests[1][1]["url"] - # Assert the response + # Parse URLs to compare query params order-independently + # Parse first URL + parsed_first = urlparse(str(first_transformed_url)) + first_params = parse_qs(parsed_first.query) + + # Parse second URL + parsed_second = urlparse(str(second_transformed_url)) + second_params = parse_qs(parsed_second.query) + + # Expected values (parse_qs decodes + as space) + expected_first_params = {"q": ["bob barker"], "setLang": ["en-US"], "mkt": ["en-US"]} + expected_second_params = {"setLang": ["en-US"], "mkt": ["en-US"]} + + # Assert the response - compare base URL and params separately assert ( - first_transformed_url - == "https://api.bing.microsoft.com/v7.0/search?q=bob+barker&setLang=en-US&mkt=en-US" - and second_transformed_url - == "https://api.bing.microsoft.com/v7.0/search?setLang=en-US&mkt=en-US" + parsed_first.scheme == "https" + and parsed_first.netloc == "api.bing.microsoft.com" + and parsed_first.path == "/v7.0/search" + and first_params == expected_first_params + and parsed_second.scheme == "https" + and parsed_second.netloc == "api.bing.microsoft.com" + and parsed_second.path == "/v7.0/search" + and second_params == expected_second_params ) From 2bc4c4359db749f030ce19666cd4339ae54412c4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 09:04:26 +0530 Subject: [PATCH 10/13] Add supports_web_search for sonnet 4 --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 034b80a58c..ac9ad81919 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8201,6 +8201,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "supports_web_search": true, "tool_use_system_prompt_tokens": 159 }, "claude-sonnet-4-5": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 034b80a58c..ac9ad81919 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8201,6 +8201,7 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, + "supports_web_search": true, "tool_use_system_prompt_tokens": 159 }, "claude-sonnet-4-5": { From 4d6b7699cc6e512ed2368eb7a04f66372a30c6f4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 09:16:24 +0530 Subject: [PATCH 11/13] Fix sonnet 3.7 tests --- .../test_anthropic_completion.py | 10 +++++----- tests/local_testing/test_caching.py | 20 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index e240437178..8630ba6561 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -489,7 +489,7 @@ class TestAnthropicCompletion(BaseLLMChatTest, BaseAnthropicChatTest): def get_base_completion_call_args_with_thinking(self) -> dict: return { - "model": "anthropic/claude-3-7-sonnet-latest", + "model": "anthropic/claude-sonnet-4-5-20250929", "thinking": {"type": "enabled", "budget_tokens": 16000}, } @@ -701,7 +701,7 @@ def test_anthropic_tool_with_image(): ] result = prompt_factory( - model="claude-3-5-sonnet-20240620", + model="claude-sonnet-4-5-20250929", messages=messages, custom_llm_provider="anthropic", ) @@ -761,7 +761,7 @@ def test_anthropic_map_openai_params_tools_and_json_schema(): mapped_params = litellm.AnthropicConfig().map_openai_params( non_default_params=args["non_default_params"], optional_params={}, - model="claude-3-5-sonnet-20240620", + model="claude-sonnet-4-5-20250929", drop_params=False, ) @@ -803,7 +803,7 @@ def test_anthropic_map_openai_params_tools_with_defs(): mapped_params = litellm.AnthropicConfig().map_openai_params( non_default_params=args["non_default_params"], optional_params={}, - model="claude-3-5-sonnet-20240620", + model="claude-sonnet-4-5-20250929", drop_params=False, ) @@ -1133,7 +1133,7 @@ def test_anthropic_custom_headers(): with patch.object(client, "post") as mock_post: try: resp = completion( - model="claude-3-5-sonnet-20240620", + model="claude-sonnet-4-5-20250929", headers={"anthropic-beta": "computer-use-2025-01-24"}, messages=[ {"role": "user", "content": "What is the capital of France?"} diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 7fb57cef9e..3c421e1509 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -2647,13 +2647,13 @@ def test_caching_with_reasoning_content(): litellm.cache = Cache() response_1 = completion( - model="anthropic/claude-3-7-sonnet-latest", + model="anthropic/claude-sonnet-4-5-20250929", messages=messages, thinking={"type": "enabled", "budget_tokens": 1024}, ) response_2 = completion( - model="anthropic/claude-3-7-sonnet-latest", + model="anthropic/claude-sonnet-4-5-20250929", messages=messages, thinking={"type": "enabled", "budget_tokens": 1024}, ) @@ -2671,14 +2671,14 @@ def test_caching_reasoning_args_miss(): # test in memory cache litellm.set_verbose = True litellm.cache = Cache() response1 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, reasoning_effort="low", mock_response="My response", ) response2 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, mock_response="My response", @@ -2697,14 +2697,14 @@ def test_caching_reasoning_args_hit(): # test in memory cache litellm.set_verbose = True litellm.cache = Cache() response1 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, reasoning_effort="low", mock_response="My response", ) response2 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, reasoning_effort="low", @@ -2724,14 +2724,14 @@ def test_caching_thinking_args_miss(): # test in memory cache litellm.set_verbose = True litellm.cache = Cache() response1 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, thinking={"type": "enabled", "budget_tokens": 1024}, mock_response="My response", ) response2 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, mock_response="My response", @@ -2750,14 +2750,14 @@ def test_caching_thinking_args_hit(): # test in memory cache litellm.set_verbose = True litellm.cache = Cache() response1 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, thinking={"type": "enabled", "budget_tokens": 1024}, mock_response="My response", ) response2 = completion( - model="claude-3-7-sonnet-latest", + model="claude-4-sonnet-20250514", messages=messages, caching=True, thinking={"type": "enabled", "budget_tokens": 1024}, From 631400cb17a58fe183ca1da16cb8ea66462574ea Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 09:28:05 +0530 Subject: [PATCH 12/13] Fix anthropic responses --- .../llm_responses_api_testing/test_anthropic_responses_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py index 47ea3f7aa5..213c96190e 100644 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ b/tests/llm_responses_api_testing/test_anthropic_responses_api.py @@ -79,7 +79,7 @@ def test_multiturn_tool_calls(): ], 'type': 'message' }], - model='anthropic/claude-sonnet-4-5', + model='anthropic/claude-4-sonnet-20250514', instructions='You are a helpful coding assistant.', tools=[shell_tool] ) @@ -105,7 +105,7 @@ def test_multiturn_tool_calls(): # Use await with asyncio.run for the async function follow_up_response = litellm.responses( - model='anthropic/claude-sonnet-4-5', + model='anthropic/claude-4-sonnet-20250514', previous_response_id=response_id, input=[{ 'type': 'function_call_output', From 01148511fbef694301f40a0759ba9104c62ccff3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 20 Feb 2026 09:28:49 +0530 Subject: [PATCH 13/13] Fix: litellm/tests/llm_responses_api_testing/test_anthropic_responses_api.py --- tests/proxy_e2e_anthropic_messages_tests/test_config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml index fbbb6d4114..72be11468f 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml +++ b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml @@ -50,4 +50,7 @@ model_list: vertex_ai_location: "asia-southeast1" general_settings: - forward_client_headers_to_llm_api: true \ No newline at end of file + forward_client_headers_to_llm_api: true + +litellm_settings: + drop_params: true \ No newline at end of file