From 2c733c00f56782c9310f710d65615bfbc89b0af7 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 15 May 2026 15:44:28 -0700 Subject: [PATCH] chore(ci): modernize model references in tests and configs (#27856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: modernize models used in CircleCI e2e test suites Replaces obsolete models (gpt-4o, gpt-4o-mini, gpt-3.5-turbo, claude-3-5-sonnet-20240620, claude-sonnet-4-20250514) with current equivalents across the e2e_openai_endpoints and proxy_e2e_anthropic_messages_tests CircleCI jobs. - gpt-4o -> gpt-5.5 (responses API e2e tests) - gpt-4o-mini -> gpt-5-mini (websocket responses, oai_misc_config) - gpt-4o-mini-2024-07-18 -> gpt-4.1-mini-2025-04-14 (fine-tuning, still actively fine-tunable) - gpt-4 / gpt-3.5-turbo target_model_names example -> gpt-5.5 / gpt-5-mini - bedrock claude-3-5-sonnet-20240620 batch entry -> haiku-4-5-20251001 (also aligning oai_misc_config model_name with what test_bedrock_batches_api.py actually requests) - bedrock claude-sonnet-4-20250514 (deprecated, retires 2026-06-15) -> claude-sonnet-4-5-20250929 * test: point bedrock-claude-sonnet-4 alias at Sonnet 4.6, not 4.5 Greptile/Cursor flagged that after the previous commit, the bedrock-claude-sonnet-4 alias collided with bedrock-claude-sonnet-4.5 (both pointed to claude-sonnet-4-5-20250929). Rename to bedrock-claude-sonnet-4.6 and point it at the Sonnet 4.6 Bedrock ID (us.anthropic.claude-sonnet-4-6, already in the litellm model registry) so the alias name matches the underlying model version. * test: modernize models across remaining CI-mounted configs & tests Expands the modernization sweep to all CircleCI-mounted proxy configs and to test directories where the model literal is a fixture/route key (not the test's subject). Config changes: - proxy_server_config.yaml: bump gpt-3.5-turbo / gpt-3.5-turbo-1106 / gpt-4o / gemini-1.5-flash / dall-e-3 underlying models; rename gpt-3.5-turbo-end-user-test alias to gpt-5-mini-end-user-test; bump text-embedding-ada-002 underlying to text-embedding-3-small. User- facing aliases (gpt-3.5-turbo, gpt-4, text-embedding-ada-002, etc.) preserved for backward compatibility with tests. - simple_config.yaml, otel_test_config.yaml, spend_tracking_config.yaml: bump gpt-3.5-turbo underlying to gpt-5-mini. - pass_through_config.yaml: claude-3-5-sonnet / claude-3-7-sonnet / claude-3-haiku entries replaced with claude-sonnet-4-5 / claude- haiku-4-5 / claude-opus-4-7. - oai_misc_config.yaml: align alias name with the gpt-5-mini rename. Test changes (proactive: claude-sonnet-4-20250514 / claude-opus-4- 20250514 retire 2026-06-15): - tests/llm_translation/test_anthropic_completion.py: bump 3 references + paired Vertex AI ID to claude-sonnet-4-5. - tests/llm_translation/test_optional_params.py: bump 2 references. - tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py and test_bedrock_anthropic_messages_test.py: bump router fixtures using the deprecated model IDs. - tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py: modernize docstring examples. - tests/test_end_users.py: update references to renamed alias. * test: modernize placeholder model literals in router_unit_tests Mass replace_all on fixture/placeholder model literals across the router_unit_tests/ suite (model name is a routing key / label, not the test subject). Sub-agent sweep so far — additional commits will follow for logging_callback_tests/, enterprise/, top-level tests/test_*.py, and other CI-mounted dirs. Mappings applied: - gpt-3.5-turbo -> gpt-5-mini - gpt-4 (bare) -> gpt-5.5 - gpt-4o (bare) -> gpt-5 - text-embedding-ada-002 -> text-embedding-3-small - claude-3-sonnet-20240229 / claude-3-opus-20240229 / claude-3-haiku-20240307 / claude-3-5-sonnet-20240620 -> claude-sonnet-4-5-20250929 / claude-opus-4-7 / claude-haiku-4-5-20251001 as appropriate Explicitly preserved: - gpt-4o-mini-* variants (transcribe, tts, etc.) where they're current - gpt-4-turbo / gpt-4-vision-preview / gpt-4-0613 (subject literals) - JSONL batch body literals - Mock LLM response model fields (must match upstream) - Fake/mock identifiers * test: modernize placeholder model literals across remaining CI suites Sub-agent sweep across logging_callback_tests/, guardrails_tests/, enterprise/, pass_through_unit_tests/, otel_tests/, llm_responses_api_testing/, batches_tests/, spend_tracking_tests/, litellm_utils_tests/, unified_google_tests/, and a few top-level tests/test_*.py files where the model literal is a fixture or placeholder (router model_list, mock standard logging payload, mock callback data) rather than the test's subject. Mappings applied (see scope notes below): - gpt-3.5-turbo -> gpt-5-mini - gpt-4 (bare) -> gpt-5.5 - gpt-4o (bare) -> gpt-5.5 (corrected from initial gpt-5 — bare gpt-5 is not a valid OpenAI alias; only gpt-5.5 / gpt-5.4 / gpt-5.2-codex / gpt-5-mini exist) - gpt-4o-mini (bare) -> gpt-5-mini - text-embedding-ada-002 -> text-embedding-3-small - claude-3-sonnet-20240229 -> claude-sonnet-4-5-20250929 - claude-3-opus-20240229 -> claude-opus-4-7 - claude-3-haiku-20240307 -> claude-haiku-4-5-20251001 - claude-3-5-sonnet-20240620/20241022 -> claude-sonnet-4-5-20250929 - claude-3-7-sonnet-20250219 -> claude-sonnet-4-6 - gemini-1.5-flash -> gemini-2.5-flash - gemini-1.5-pro -> gemini-2.5-pro Explicitly preserved (not modernized): - llm_translation/ tests where model is the SUBJECT (provider-specific translation/transformation logic). Only the deprecated 20250514 references were already bumped in a prior commit. - Cost-calc / tokenizer subject tests in test_utils.py (skip-ranges documented by the sub-agent). - Bedrock model IDs in test_health_check.py path-stripping tests. - JSONL batch request bodies and mock LLM response bodies (must match upstream literal). - Langfuse expected-request-body JSON fixtures (cost values are exact- match-asserted; changing the model would shift response_cost). - gpt-3.5-turbo-instruct (text-completion endpoint; no modern OpenAI equivalent). - Top-level tests calling the proxy through user-facing aliases (gpt-3.5-turbo, gpt-4, text-embedding-ada-002, dall-e-3) — aliases in proxy_server_config.yaml stay; only the underlying model was bumped. - tests/test_gpt5_azure_temperature_support.py (the test's whole point is model-name handling). - Fake / mock / openai/fake identifiers. Notable side fixes: - test_spend_accuracy_tests.py: UPSTREAM_MODEL now matches what spend_tracking_config.yaml's proxy actually routes to (gpt-5-mini), resolving a latent inconsistency. - proxy_server_config.yaml: bare `gpt-5` alias renamed to `gpt-5.5` (bare gpt-5 is not a valid OpenAI alias). - test_batches_logging_unit_tests.py: explicit_models list entries kept distinct (gpt-5-mini + gpt-5.5) after bulk rename. * test: fix CI failures from model modernization sweep CI surfaced 4 categories of regression from the bulk modernization: 1. Azure deployment names are customer-specific. Reverted: - tests/litellm_utils_tests/test_health_check.py: azure/text- embedding-3-small -> azure/text-embedding-ada-002 (the CI Azure account does not have a text-embedding-3-small deployment). - tests/logging_callback_tests/test_custom_callback_router.py: same revert for two router fixtures driving aembedding. 2. gpt-5 family does not accept temperature != 1. Tests that pass a custom temperature swapped from gpt-5-mini to gpt-4.1-mini (modern non-reasoning OpenAI mini that still accepts temperature/logprobs): - tests/logging_callback_tests/test_datadog.py - tests/logging_callback_tests/test_langsmith_unit_test.py - tests/logging_callback_tests/test_otel_logging.py 3. proxy_server_config.yaml's gpt-3.5-turbo-large alias was routing to gpt-5.5 (a reasoning model that rejects logprobs). The proxy test tests/test_openai_endpoints.py::test_chat_completion_streaming exercises logprobs/top_logprobs through that alias. Bumped the underlying model to gpt-4.1 (non-reasoning, still modern). 4. tests/logging_callback_tests/test_gcs_pub_sub.py asserts against a pinned JSON fixture (gcs_pub_sub_body/spend_logs_payload.json) with hardcoded model="gpt-4o" and a model-specific spend value. Reverted the litellm.acompletion calls in the test to model="gpt-4o" so the fixture's exact-match assertions still hold. 5. tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py: anthropic.messages.create routing to openai/gpt-5-mini returned an empty content[0] with max_tokens=100 (reasoning-token consumption). Swapped to openai/gpt-4.1-mini. * test: fix Assistants API model + 2 cursor[bot] review nits 1. pass_through_unit_tests/test_custom_logger_passthrough.py: gpt-5.5 isn't accepted by the /v1/assistants endpoint ("unsupported_model"). Switch to gpt-4.1-mini (modern, Assistants- API-supported, non-reasoning). 2. example_config_yaml/pass_through_config.yaml: the previous sweep bumped the claude-3-7-sonnet alias to claude-opus-4-7, which is a tier change (Sonnet -> Opus). Map to claude-sonnet-4-6 to keep the Sonnet tier intact. (Cursor bugbot review.) 3. example_config_yaml/simple_config.yaml: model_name was left as gpt-3.5-turbo while the underlying was bumped to gpt-5-mini, which muddles the "simple" example. Make both sides gpt-5-mini so the most basic example is a straight 1:1 mapping again. (Cursor bugbot review.) * fix: revert gpt-4/gpt-3.5-turbo alias underlying to non-reasoning models tests/test_openai_endpoints.py::test_completion calls the proxy alias "gpt-4" with temperature=0, and other tests call gpt-3.5-turbo with custom temperature / logprobs / the legacy /v1/completions endpoint. The earlier modernization mapped both aliases to gpt-5.5 / gpt-5-mini, which are reasoning models that reject temperature != 1 and don't expose /v1/completions. Map the aliases to gpt-4.1 / gpt-4.1-mini (modern non-reasoning OpenAI models) instead — keeps user-facing aliases preserved while picking a current underlying that still supports the parameters/endpoints the tests exercise. --- .../example_config_yaml/oai_misc_config.yaml | 10 +- .../example_config_yaml/otel_test_config.yaml | 4 +- .../pass_through_config.yaml | 14 +- .../example_config_yaml/simple_config.yaml | 4 +- .../spend_tracking_config.yaml | 2 +- proxy_server_config.yaml | 34 +-- .../test_batches_logging_unit_tests.py | 14 +- .../test_prometheus_logging_callbacks.py | 104 ++++----- .../integrations/test_prometheus.py | 26 +-- .../test_prometheus_unit_tests.py | 28 +-- .../proxy/hooks/test_managed_files.py | 26 +-- .../test_project_endpoints_prisma.py | 66 +++--- .../guardrails_tests/test_akto_guardrails.py | 10 +- .../test_bedrock_guardrails.py | 56 ++--- .../test_dynamoai_guardrails.py | 4 +- .../test_guardrail_load_balancing.py | 4 +- .../test_guardrails_config.py | 4 +- tests/guardrails_tests/test_lakera_v2.py | 20 +- tests/guardrails_tests/test_presidio_pii.py | 6 +- .../test_tracing_guardrails.py | 14 +- .../litellm_utils_tests/test_health_check.py | 40 ++-- tests/litellm_utils_tests/test_utils.py | 66 +++--- .../base_responses_api.py | 6 +- ...t_base_responses_api_streaming_iterator.py | 20 +- .../test_openai_responses_api.py | 72 +++---- .../test_anthropic_completion.py | 8 +- tests/llm_translation/test_optional_params.py | 4 +- .../create_mock_standard_logging_payload.py | 8 +- .../create_mock_standard_logging_payload.py | 8 +- tests/logging_callback_tests/test_alerting.py | 40 ++-- .../test_amazing_s3_logs.py | 16 +- .../test_assemble_streaming_responses.py | 12 +- .../test_bedrock_knowledgebase_hook.py | 20 +- .../test_custom_callback_router.py | 20 +- tests/logging_callback_tests/test_datadog.py | 12 +- .../test_datadog_llm_obs.py | 6 +- .../test_generic_api_callback.py | 26 +-- .../test_langfuse_unit_tests.py | 4 +- .../test_langsmith_unit_test.py | 10 +- .../test_logging_redaction_e2e_test.py | 16 +- .../test_opentelemetry_unit_tests.py | 2 +- .../test_otel_logging.py | 8 +- .../test_pagerduty_alerting.py | 10 +- tests/logging_callback_tests/test_posthog.py | 8 +- .../logging_callback_tests/test_spend_logs.py | 20 +- .../logging_callback_tests/test_sqs_logger.py | 8 +- .../test_standard_logging_payload.py | 26 +-- ...tandard_logging_payload_excluded_fields.py | 2 +- .../test_token_counting.py | 6 +- .../test_unit_test_litellm_logging.py | 2 +- .../test_unit_tests_init_callbacks.py | 10 +- .../test_e2e_openai_responses_api.py | 10 +- .../test_openai_batches_endpoint.py | 4 +- .../test_openai_fine_tuning.py | 2 +- .../test_responses_websocket_proxy_e2e.py | 4 +- tests/otel_tests/test_e2e_model_access.py | 34 +-- tests/otel_tests/test_guardrails.py | 2 +- tests/otel_tests/test_otel.py | 6 +- tests/otel_tests/test_prometheus.py | 2 +- ...ase_anthropic_messages_tool_search_test.py | 6 +- .../test_anthropic_messages_passthrough.py | 20 +- .../test_anthropic_messages_tool_search.py | 2 +- .../test_bedrock_anthropic_messages_test.py | 22 +- .../test_custom_logger_passthrough.py | 2 +- .../test_pass_through_unit_tests.py | 10 +- .../test_unit_test_anthropic_pass_through.py | 12 +- .../test_config.yaml | 4 +- .../create_mock_standard_logging_payload.py | 8 +- .../test_completion_no_copy.py | 8 +- .../test_default_deployment_copy.py | 4 +- .../test_get_model_list_alias_optimization.py | 18 +- .../test_pre_call_checks_optimization.py | 16 +- .../test_prompt_management_check.py | 10 +- .../test_router_acancel_batch.py | 6 +- .../test_router_batch_utils.py | 4 +- .../test_router_cooldown_utils.py | 20 +- .../test_router_embedding_headers.py | 58 ++--- .../test_router_embedding_integration.py | 22 +- .../test_router_endpoints.py | 28 +-- .../test_router_handle_error.py | 16 +- .../test_router_helper_utils.py | 198 +++++++++--------- .../test_router_index_management.py | 60 +++--- .../test_router_prompt_caching.py | 2 +- .../test_ocr_spend_tracking.py | 4 +- .../test_spend_accuracy_tests.py | 2 +- tests/test_end_users.py | 4 +- tests/test_litellm_proxy_responses_config.py | 2 +- tests/test_ratelimit.py | 8 +- .../test_litellm_responses_bridge.py | 4 +- 89 files changed, 805 insertions(+), 805 deletions(-) diff --git a/litellm/proxy/example_config_yaml/oai_misc_config.yaml b/litellm/proxy/example_config_yaml/oai_misc_config.yaml index 45c6e44132..16cc69c19a 100644 --- a/litellm/proxy/example_config_yaml/oai_misc_config.yaml +++ b/litellm/proxy/example_config_yaml/oai_misc_config.yaml @@ -1,7 +1,7 @@ model_list: - - model_name: gpt-3.5-turbo-end-user-test + - model_name: gpt-5-mini-end-user-test litellm_params: - model: gpt-3.5-turbo + model: gpt-5-mini region_name: "eu" model_info: id: "1" @@ -18,9 +18,9 @@ model_list: litellm_params: model: "groq/*" api_key: os.environ/GROQ_API_KEY - - model_name: bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0 + - model_name: bedrock/batch-us.anthropic.claude-haiku-4-5-20251001-v1:0 litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 ######################################################### ########## batch specific params ######################## s3_bucket_name: litellm-proxy @@ -39,7 +39,7 @@ litellm_settings: num_retries: 5 request_timeout: 600 telemetry: False - context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] + context_window_fallbacks: [{"gpt-5-mini": ["gpt-5.5"]}] default_team_settings: - team_id: team-1 success_callback: ["langfuse"] diff --git a/litellm/proxy/example_config_yaml/otel_test_config.yaml b/litellm/proxy/example_config_yaml/otel_test_config.yaml index dc61286573..c05e2b1b5d 100644 --- a/litellm/proxy/example_config_yaml/otel_test_config.yaml +++ b/litellm/proxy/example_config_yaml/otel_test_config.yaml @@ -1,7 +1,7 @@ model_list: - model_name: fake-openai-endpoint litellm_params: - model: openai/gpt-3.5-turbo + model: openai/gpt-5-mini api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ tags: ["teamA"] @@ -9,7 +9,7 @@ model_list: id: "team-a-model" - model_name: fake-openai-endpoint litellm_params: - model: openai/gpt-3.5-turbo + model: openai/gpt-5-mini api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ tags: ["teamB"] diff --git a/litellm/proxy/example_config_yaml/pass_through_config.yaml b/litellm/proxy/example_config_yaml/pass_through_config.yaml index a7b65b272e..373ee189f3 100644 --- a/litellm/proxy/example_config_yaml/pass_through_config.yaml +++ b/litellm/proxy/example_config_yaml/pass_through_config.yaml @@ -4,21 +4,21 @@ model_list: model: openai/fake api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: claude-3-5-sonnet-20241022 + - model_name: claude-sonnet-4-5-20250929 litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 + model: anthropic/claude-sonnet-4-5-20250929 api_key: os.environ/ANTHROPIC_API_KEY - model_name: claude-special-alias litellm_params: - model: anthropic/claude-3-haiku-20240307 + model: anthropic/claude-haiku-4-5-20251001 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: claude-3-5-sonnet-20241022 + - model_name: claude-sonnet-4-5-20250929 litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 + model: anthropic/claude-sonnet-4-5-20250929 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: claude-3-7-sonnet-20250219 + - model_name: claude-sonnet-4-6 litellm_params: - model: anthropic/claude-3-7-sonnet-20250219 + model: anthropic/claude-sonnet-4-6 api_key: os.environ/ANTHROPIC_API_KEY - model_name: anthropic/* litellm_params: diff --git a/litellm/proxy/example_config_yaml/simple_config.yaml b/litellm/proxy/example_config_yaml/simple_config.yaml index 14b39a1251..c167412ff0 100644 --- a/litellm/proxy/example_config_yaml/simple_config.yaml +++ b/litellm/proxy/example_config_yaml/simple_config.yaml @@ -1,4 +1,4 @@ model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-5-mini litellm_params: - model: gpt-3.5-turbo \ No newline at end of file + model: gpt-5-mini \ No newline at end of file diff --git a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml index 6c2276c285..dfed2194b5 100644 --- a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml +++ b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml @@ -1,7 +1,7 @@ model_list: - model_name: fake-openai-endpoint litellm_params: - model: openai/gpt-3.5-turbo + model: openai/gpt-5-mini api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index d9838c852a..da37eb3428 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -1,28 +1,28 @@ model_list: - - model_name: gpt-3.5-turbo-end-user-test + - model_name: gpt-5-mini-end-user-test litellm_params: - model: gpt-3.5-turbo + model: gpt-5-mini region_name: "eu" model_info: id: "1" - - model_name: gpt-3.5-turbo-end-user-test + - model_name: gpt-5-mini-end-user-test litellm_params: - model: openai/gpt-4.1-mini + model: openai/gpt-5-mini api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - model_name: gpt-3.5-turbo litellm_params: model: openai/gpt-4.1-mini api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - model_name: gpt-3.5-turbo-large - litellm_params: - model: "gpt-3.5-turbo-1106" + litellm_params: + model: "gpt-4.1" api_key: os.environ/OPENAI_API_KEY rpm: 480 timeout: 300 stream_timeout: 60 - model_name: gpt-4 litellm_params: - model: openai/gpt-4.1-mini + model: openai/gpt-4.1 api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault rpm: 480 timeout: 300 @@ -32,21 +32,21 @@ model_list: model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 input_cost_per_second: 0.000420 - model_name: text-embedding-ada-002 - litellm_params: - model: openai/text-embedding-ada-002 + litellm_params: + model: openai/text-embedding-3-small api_key: os.environ/OPENAI_API_KEY model_info: mode: embedding - base_model: text-embedding-ada-002 + base_model: text-embedding-3-small - model_name: dall-e-2 # dall-e-2 and dall-e-3 were deprecated 2026-05-12; alias to gpt-image-1 litellm_params: model: openai/gpt-image-1 - - model_name: openai-dall-e-3 + - model_name: openai-dall-e-3 # dall-e-3 deprecated 2026-05-12; underlying now gpt-image-1 litellm_params: - model: dall-e-3 + model: gpt-image-1 - model_name: fake-openai-endpoint litellm_params: - model: openai/gpt-3.5-turbo + model: openai/gpt-5-mini api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ - model_name: fake-openai-endpoint-2 @@ -139,13 +139,13 @@ model_list: model: openai/my-fake-model api_key: my-fake-key api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/ - - model_name: gemini-1.5-flash + - model_name: gemini-2.5-flash litellm_params: - model: gemini/gemini-1.5-flash + model: gemini/gemini-2.5-flash api_key: os.environ/GOOGLE_API_KEY - - model_name: gpt-4o + - model_name: gpt-5.5 litellm_params: - model: gpt-4o + model: gpt-5.5 api_key: os.environ/OPENAI_API_KEY diff --git a/tests/batches_tests/test_batches_logging_unit_tests.py b/tests/batches_tests/test_batches_logging_unit_tests.py index 0b471bbe75..a0c9469378 100644 --- a/tests/batches_tests/test_batches_logging_unit_tests.py +++ b/tests/batches_tests/test_batches_logging_unit_tests.py @@ -215,7 +215,7 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos # Create logging object logging_obj = Logging( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "test"}], stream=False, call_type=CallTypes.aretrieve_batch.value, @@ -233,7 +233,7 @@ async def test_batch_retrieve_cost_tracking_with_completed_batch_no_explicit_cos completion_tokens=50, total_tokens=150, ) - expected_models = ["gpt-4o-mini"] + expected_models = ["gpt-5-mini"] with patch( "litellm.litellm_core_utils.litellm_logging._handle_completed_batch", @@ -299,7 +299,7 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): # Create logging object logging_obj = Logging( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "test"}], stream=False, call_type=CallTypes.aretrieve_batch.value, @@ -317,7 +317,7 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data(): completion_tokens=100, total_tokens=300, ) - explicit_models = ["gpt-4o-mini", "gpt-3.5-turbo"] + explicit_models = ["gpt-5-mini", "gpt-5.5"] with patch( "litellm.litellm_core_utils.litellm_logging._handle_completed_batch", @@ -393,7 +393,7 @@ async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batc # Create logging object logging_obj = Logging( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "test"}], stream=False, call_type=CallTypes.aretrieve_batch.value, @@ -468,7 +468,7 @@ async def test_batch_retrieve_cost_tracking_with_partial_explicit_data(): # Create logging object logging_obj = Logging( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "test"}], stream=False, call_type=CallTypes.aretrieve_batch.value, @@ -489,7 +489,7 @@ async def test_batch_retrieve_cost_tracking_with_partial_explicit_data(): completion_tokens=75, total_tokens=225, ) - expected_models = ["gpt-4o-mini"] + expected_models = ["gpt-5-mini"] with patch( "litellm.litellm_core_utils.litellm_logging._handle_completed_batch", diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index e6c76fa7cd..f8bac82058 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -58,9 +58,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", custom_llm_provider="openai", @@ -109,7 +109,7 @@ def test_safe_get_remaining_budget(prometheus_logger): async def test_async_log_success_event(prometheus_logger): standard_logging_object = create_standard_logging_payload() kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "stream": True, "litellm_params": { "metadata": { @@ -208,7 +208,7 @@ def test_increment_token_metrics(prometheus_logger): end_user_id="user1", user_api_key="key1", user_api_key_alias="alias1", - model="gpt-3.5-turbo", + model="gpt-5-mini", user_api_team="team1", user_api_team_alias="team_alias1", user_id="user1", @@ -226,7 +226,7 @@ def test_increment_token_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model=None, - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_tokens_metric.labels().inc.assert_called_once_with(100) @@ -242,7 +242,7 @@ def test_increment_token_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model=None, - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_input_tokens_metric.labels().inc.assert_called_once_with( @@ -260,7 +260,7 @@ def test_increment_token_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model=None, - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_output_tokens_metric.labels().inc.assert_called_once_with( @@ -403,7 +403,7 @@ def test_set_latency_metrics(prometheus_logger): prometheus_logger._set_latency_metrics( kwargs=kwargs, - model="gpt-3.5-turbo", + model="gpt-5-mini", user_api_key="key1", user_api_key_alias="alias1", user_api_team="team1", @@ -422,7 +422,7 @@ def test_set_latency_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model="openai-gpt", - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_llm_api_time_to_first_token_metric.labels().observe.assert_called_once_with( @@ -440,7 +440,7 @@ def test_set_latency_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model="openai-gpt", - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_llm_api_latency_metric.labels().observe.assert_called_once_with( @@ -458,7 +458,7 @@ def test_set_latency_metrics(prometheus_logger): org_id=None, org_alias=None, requested_model="openai-gpt", - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", ) prometheus_logger.litellm_request_total_latency_metric.labels().observe.assert_called_once_with( @@ -497,7 +497,7 @@ def test_set_latency_metrics_missing_timestamps(prometheus_logger): # This should not raise an exception prometheus_logger._set_latency_metrics( kwargs=kwargs, - model="gpt-3.5-turbo", + model="gpt-5-mini", user_api_key="key1", user_api_key_alias="alias1", user_api_team="team1", @@ -544,7 +544,7 @@ def test_set_latency_metrics_missing_api_call_start(prometheus_logger): # This should not raise an exception prometheus_logger._set_latency_metrics( kwargs=kwargs, - model="gpt-3.5-turbo", + model="gpt-5-mini", user_api_key="key1", user_api_key_alias="alias1", user_api_team="team1", @@ -584,7 +584,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger): end_user_id="user1", user_api_key="key1", user_api_key_alias="alias1", - model="gpt-3.5-turbo", + model="gpt-5-mini", user_api_team="team1", user_api_team_alias="team_alias1", user_id="user1", @@ -602,7 +602,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", api_provider="openai", client_ip=None, @@ -621,7 +621,7 @@ def test_increment_top_level_request_and_spend_metrics(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", api_provider="openai", client_ip=None, @@ -635,7 +635,7 @@ async def test_async_log_failure_event(prometheus_logger): # NOTE: almost all params for this metric are read from standard logging payload standard_logging_object = create_standard_logging_payload() kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": { "custom_llm_provider": "openai", }, @@ -664,7 +664,7 @@ async def test_async_log_failure_event(prometheus_logger): end_user=None, hashed_api_key="test_hash", api_key_alias="test_alias", - model="gpt-3.5-turbo", + model="gpt-5-mini", team="test_team", team_alias="test_team_alias", user="test_user", @@ -674,7 +674,7 @@ async def test_async_log_failure_event(prometheus_logger): # deployment should be marked in partial outage prometheus_logger.set_deployment_partial_outage.assert_called_once_with( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -686,7 +686,7 @@ async def test_async_log_failure_event(prometheus_logger): prometheus_logger.litellm_deployment_failure_responses.labels.call_args.kwargs ) expected_failure_labels = { - "litellm_model_name": "gpt-3.5-turbo", + "litellm_model_name": "gpt-5-mini", "model_id": "model-123", "api_base": "https://api.openai.com", "api_provider": "openai", @@ -712,7 +712,7 @@ async def test_async_log_failure_event(prometheus_logger): prometheus_logger.litellm_deployment_total_requests.labels.call_args.kwargs ) expected_total_labels = { - "litellm_model_name": "gpt-3.5-turbo", + "litellm_model_name": "gpt-5-mini", "model_id": "model-123", "api_base": "https://api.openai.com", "api_provider": "openai", @@ -788,10 +788,10 @@ async def test_async_post_call_failure_hook(prometheus_logger): prometheus_logger.litellm_proxy_total_requests_metric = MagicMock() # Create test data - request_data = {"model": "gpt-3.5-turbo"} + request_data = {"model": "gpt-5-mini"} original_exception = litellm.RateLimitError( - message="Test error", llm_provider="openai", model="gpt-3.5-turbo" + message="Test error", llm_provider="openai", model="gpt-5-mini" ) user_api_key_dict = UserAPIKeyAuth( @@ -822,7 +822,7 @@ async def test_async_post_call_failure_hook(prometheus_logger): team_alias="test_team_alias", org_id=None, org_alias=None, - requested_model="gpt-3.5-turbo", + requested_model="gpt-5-mini", exception_status="429", exception_class="Openai.RateLimitError", route=user_api_key_dict.request_route, @@ -837,7 +837,7 @@ async def test_async_post_call_failure_hook(prometheus_logger): end_user=None, hashed_api_key="test_key", api_key_alias="test_alias", - requested_model="gpt-3.5-turbo", + requested_model="gpt-5-mini", team="test_team", team_alias="test_team_alias", org_id=None, @@ -865,7 +865,7 @@ async def test_async_post_call_success_hook(prometheus_logger): prometheus_logger.litellm_proxy_total_requests_metric = MagicMock() # Create test data - data = {"model": "gpt-3.5-turbo"} + data = {"model": "gpt-5-mini"} user_api_key_dict = UserAPIKeyAuth( api_key="test_key", @@ -909,7 +909,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Create test data request_kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": { "custom_llm_provider": "openai", "metadata": {"model_info": {"id": "model-123"}}, @@ -946,7 +946,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): model_group="my_custom_model_group", # model_group / requested model from create_standard_logging_payload() api_provider="openai", # llm provider api_base="https://api.openai.com", # api base - litellm_model_name="gpt-3.5-turbo", # actual model used - litellm model name + litellm_model_name="gpt-5-mini", # actual model used - litellm model name hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], model_id="model-123", @@ -962,7 +962,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], api_provider="openai", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_group="my_custom_model_group", model_id="model-123", ) @@ -973,7 +973,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Verify deployment healthy state prometheus_logger.set_deployment_healthy.assert_called_once_with( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -981,7 +981,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Verify success responses metric prometheus_logger.litellm_deployment_success_responses.labels.assert_called_once_with( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -997,7 +997,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Verify total requests metric prometheus_logger.litellm_deployment_total_requests.labels.assert_called_once_with( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -1013,7 +1013,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): # Verify latency per output token metric prometheus_logger.litellm_deployment_latency_per_output_token.labels.assert_called_once_with( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -1029,7 +1029,7 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): api_key_alias=standard_logging_payload["metadata"]["user_api_key_alias"], api_provider="openai", hashed_api_key=standard_logging_payload["metadata"]["user_api_key_hash"], - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_group="my_custom_model_group", model_id="model-123", ) @@ -1045,9 +1045,9 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): async def test_log_success_fallback_event(prometheus_logger): prometheus_logger.litellm_deployment_successful_fallbacks = MagicMock() - original_model_group = "gpt-3.5-turbo" + original_model_group = "gpt-5-mini" kwargs = { - "model": "gpt-4", + "model": "gpt-5.5", "metadata": { "user_api_key_hash": "test_hash", "user_api_key_alias": "test_alias", @@ -1056,7 +1056,7 @@ async def test_log_success_fallback_event(prometheus_logger): }, } original_exception = litellm.RateLimitError( - message="Test error", llm_provider="openai", model="gpt-3.5-turbo" + message="Test error", llm_provider="openai", model="gpt-5-mini" ) await prometheus_logger.log_success_fallback_event( @@ -1067,7 +1067,7 @@ async def test_log_success_fallback_event(prometheus_logger): prometheus_logger.litellm_deployment_successful_fallbacks.labels.assert_called_once_with( requested_model=original_model_group, - fallback_model="gpt-4", + fallback_model="gpt-5.5", hashed_api_key="test_hash", api_key_alias="test_alias", team="test_team", @@ -1083,9 +1083,9 @@ async def test_log_success_fallback_event(prometheus_logger): async def test_log_failure_fallback_event(prometheus_logger): prometheus_logger.litellm_deployment_failed_fallbacks = MagicMock() - original_model_group = "gpt-3.5-turbo" + original_model_group = "gpt-5-mini" kwargs = { - "model": "gpt-4", + "model": "gpt-5.5", "metadata": { "user_api_key_hash": "test_hash", "user_api_key_alias": "test_alias", @@ -1094,7 +1094,7 @@ async def test_log_failure_fallback_event(prometheus_logger): }, } original_exception = litellm.RateLimitError( - message="Test error", llm_provider="openai", model="gpt-3.5-turbo" + message="Test error", llm_provider="openai", model="gpt-5-mini" ) await prometheus_logger.log_failure_fallback_event( @@ -1105,7 +1105,7 @@ async def test_log_failure_fallback_event(prometheus_logger): prometheus_logger.litellm_deployment_failed_fallbacks.labels.assert_called_once_with( requested_model=original_model_group, - fallback_model="gpt-4", + fallback_model="gpt-5.5", hashed_api_key="test_hash", api_key_alias="test_alias", team="test_team", @@ -1121,7 +1121,7 @@ def test_deployment_state_management(prometheus_logger): prometheus_logger.litellm_deployment_state = MagicMock() test_params = { - "litellm_model_name": "gpt-3.5-turbo", + "litellm_model_name": "gpt-5-mini", "model_id": "model-123", "api_base": "https://api.openai.com", "api_provider": "openai", @@ -1169,7 +1169,7 @@ def test_increment_deployment_cooled_down(prometheus_logger): ) prometheus_logger.increment_deployment_cooled_down( - litellm_model_name="gpt-3.5-turbo", + litellm_model_name="gpt-5-mini", model_id="model-123", api_base="https://api.openai.com", api_provider="openai", @@ -1177,7 +1177,7 @@ def test_increment_deployment_cooled_down(prometheus_logger): ) prometheus_logger.litellm_deployment_cooled_down.labels.assert_called_once_with( - "gpt-3.5-turbo", "model-123", "https://api.openai.com", "openai", "429" + "gpt-5-mini", "model-123", "https://api.openai.com", "openai", "429" ) mock_chain.inc.assert_called_once() @@ -1303,7 +1303,7 @@ async def test_async_log_success_event_with_top_level_metadata( ] = {} # Empty nested dict kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "stream": True, "litellm_params": { "metadata": { @@ -2081,7 +2081,7 @@ def test_get_exception_class_name(prometheus_logger): """ # Test case 1: Exception with llm_provider rate_limit_error = litellm.RateLimitError( - message="Rate limit exceeded", llm_provider="openai", model="gpt-3.5-turbo" + message="Rate limit exceeded", llm_provider="openai", model="gpt-5-mini" ) assert ( prometheus_logger._get_exception_class_name(rate_limit_error) @@ -2090,7 +2090,7 @@ def test_get_exception_class_name(prometheus_logger): # Test case 2: Exception with empty llm_provider auth_error = litellm.AuthenticationError( - message="Invalid API key", llm_provider="", model="gpt-4" + message="Invalid API key", llm_provider="", model="gpt-5.5" ) assert ( prometheus_logger._get_exception_class_name(auth_error) == "AuthenticationError" @@ -2098,7 +2098,7 @@ def test_get_exception_class_name(prometheus_logger): # Test case 3: Exception with None llm_provider context_window_error = litellm.ContextWindowExceededError( - message="Context length exceeded", llm_provider=None, model="gpt-4" + message="Context length exceeded", llm_provider=None, model="gpt-5.5" ) assert ( prometheus_logger._get_exception_class_name(context_window_error) @@ -2159,7 +2159,7 @@ def test_set_llm_deployment_success_metrics_with_label_filtering(): # Create test data request_kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": { "custom_llm_provider": "openai", "metadata": {"model_info": {"id": "model-123"}}, @@ -2310,7 +2310,7 @@ async def test_prometheus_token_metrics_with_prometheus_config(): standard_logging_payload["response_cost"] = 0.075 kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "stream": False, "litellm_params": { "metadata": { @@ -2357,7 +2357,7 @@ async def test_prometheus_token_metrics_with_prometheus_config(): expected_label_values = { "api_key_alias": "test_alias", "hashed_api_key": "test_hash", - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "team": "test_team", "team_alias": "test_team_alias", } diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py index 212c5d4a32..ebea96e215 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus.py @@ -110,9 +110,9 @@ def test_end_user_not_tracked_for_all_prometheus_metrics(): team="test_team", team_alias="test_team_alias", user="test_user", - requested_model="gpt-4", - model="gpt-4", - litellm_model_name="gpt-4", + requested_model="gpt-5.5", + model="gpt-5.5", + litellm_model_name="gpt-5.5", ) # Get all defined Prometheus metrics that include end_user in their labels @@ -199,7 +199,7 @@ def test_future_metrics_with_end_user_are_filtered(): hashed_api_key="test_key", api_key_alias="test_alias", team="test_team", - model="gpt-4", + model="gpt-5.5", ) # Test the filtering @@ -556,7 +556,7 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger): # Test data with large token count that should NOT affect request counter kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": {"metadata": {}}, "start_time": datetime.now() - timedelta(seconds=1), "end_time": datetime.now(), @@ -566,7 +566,7 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger): "prompt_tokens": 600, "completion_tokens": 399, "response_cost": 0.005, - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", "model_id": "test-model-id", "api_base": "https://api.openai.com/v1", "custom_llm_provider": "openai", @@ -605,7 +605,7 @@ async def test_request_counter_semantic_validation(mock_prometheus_logger): hashed_api_key="test-hash", api_key_alias="test-alias", team="test-team", - model="gpt-4", + model="gpt-5.5", ), response=MagicMock(), ) @@ -643,7 +643,7 @@ async def test_multiple_requests_counter_semantics(mock_prometheus_logger): for i in range(num_requests): kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": {"metadata": {}}, "start_time": datetime.now() - timedelta(seconds=1), "end_time": datetime.now(), @@ -653,7 +653,7 @@ async def test_multiple_requests_counter_semantics(mock_prometheus_logger): "prompt_tokens": tokens_per_request // 2, "completion_tokens": tokens_per_request // 2, "response_cost": 0.001, - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", "model_id": "test-model-id", "api_base": "https://api.openai.com/v1", "custom_llm_provider": "openai", @@ -707,7 +707,7 @@ async def test_streaming_request_counter_semantics(mock_prometheus_logger): from datetime import datetime, timedelta kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": {"metadata": {}}, "start_time": datetime.now() - timedelta(seconds=1), "end_time": datetime.now(), @@ -717,7 +717,7 @@ async def test_streaming_request_counter_semantics(mock_prometheus_logger): "prompt_tokens": 300, "completion_tokens": 450, "response_cost": 0.003, - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", "model_id": "test-model-id", "api_base": "https://api.openai.com/v1", "custom_llm_provider": "openai", @@ -801,7 +801,7 @@ async def test_spend_counter_semantics(mock_prometheus_logger): from datetime import datetime, timedelta kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "litellm_params": {"metadata": {}}, "start_time": datetime.now() - timedelta(seconds=1), "end_time": datetime.now(), @@ -811,7 +811,7 @@ async def test_spend_counter_semantics(mock_prometheus_logger): "prompt_tokens": 60, "completion_tokens": 40, "response_cost": 0.0015, # This should be used for spend metrics - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", "model_id": "test-model-id", "api_base": "https://api.openai.com/v1", "custom_llm_provider": "openai", diff --git a/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py b/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py index 76a5778347..55c4cbae82 100644 --- a/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py +++ b/tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py @@ -78,7 +78,7 @@ async def test_async_prometheus_success_logging_with_callbacks(prometheus_logger @compare_metrics async def op(): await litellm.acompletion( - model="claude-3-haiku-20240307", + model="claude-haiku-4-5-20251001", messages=[{"role": "user", "content": "what llm are u"}], max_tokens=10, mock_response="hi", @@ -103,9 +103,9 @@ async def test_async_prometheus_budget_logging_with_callbacks(prometheus_logger) router = litellm.Router( model_list=[ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "openai/gpt-3.5-turbo", + "model": "openai/gpt-5-mini", "api_key": "mock-key", }, } @@ -114,7 +114,7 @@ async def test_async_prometheus_budget_logging_with_callbacks(prometheus_logger) ) await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "llm?"}], mock_response="openai", metadata={ @@ -166,7 +166,7 @@ async def test_prometheus_metric_tracking(): router = Router( model_list=[ { - "model_name": "gpt-3.5-turbo", # openai model name + "model_name": "gpt-5-mini", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", "api_key": os.getenv("AZURE_AI_API_KEY"), @@ -176,9 +176,9 @@ async def test_prometheus_metric_tracking(): "model_info": {"id": "azure-model-id"}, }, { - "model_name": "gpt-3.5-turbo", # openai model name + "model_name": "gpt-5-mini", # openai model name "litellm_params": { - "model": "openai/gpt-4o-mini", + "model": "openai/gpt-5-mini", }, "model_info": {"id": "openai-model-id"}, }, @@ -192,7 +192,7 @@ async def test_prometheus_metric_tracking(): try: response = await router.acompletion( messages=[{"role": "user", "content": "Hello, how are you?"}], - model="openai/gpt-4o-mini", + model="openai/gpt-5-mini", mock_response="hi", ) print(response) @@ -252,8 +252,8 @@ async def test_router_cooldown_event_callback(): # Mock Router instance mock_router = MagicMock() mock_deployment = { - "litellm_params": {"model": "gpt-3.5-turbo"}, - "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "gpt-5-mini"}, + "model_name": "gpt-5-mini", "model_info": ModelInfo(id="test-model-id"), } mock_router.get_deployment.return_value = mock_deployment @@ -288,13 +288,13 @@ async def test_router_cooldown_event_callback(): assert len(prometheus_logger.deployment_cooled_downs) == 1 assert prometheus_logger.deployment_complete_outages[0] == [ - "gpt-3.5-turbo", + "gpt-5-mini", "test-model-id", "https://api.openai.com", "openai", ] assert prometheus_logger.deployment_cooled_downs[0] == [ - "gpt-3.5-turbo", + "gpt-5-mini", "test-model-id", "https://api.openai.com", "openai", @@ -312,8 +312,8 @@ async def test_router_cooldown_event_callback_no_prometheus(): # Mock Router instance mock_router = MagicMock() mock_deployment = { - "litellm_params": {"model": "gpt-3.5-turbo"}, - "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "gpt-5-mini"}, + "model_name": "gpt-5-mini", "model_info": ModelInfo(id="test-model-id"), } mock_router.get_deployment.return_value = mock_deployment diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 9f4ca4ed10..b9c1473924 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -392,13 +392,13 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc router = litellm.Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_info": {"id": "1234"}, }, { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_info": {"id": "5678"}, }, ], @@ -413,7 +413,7 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc ) as mock_acreate_batch: for _ in range(1000): await router.acreate_batch( - model="gpt-3.5-turbo", + model="gpt-5-mini", input_file_id=file_id, model_file_id_mapping=model_file_id_mapping, ) @@ -463,7 +463,7 @@ async def test_output_file_id_for_batch_retrieve(): "model_id": "12345679", "response_cost": 0.0, "additional_headers": {}, - "litellm_model_name": "gpt-4o", + "litellm_model_name": "gpt-5.5", "unified_batch_id": "litellm_proxy;model_id:12345679;llm_batch_id:batch_685c5e5d63988190b85bdb2147ba131d", } proxy_managed_files = _PROXY_LiteLLMManagedFiles( @@ -595,10 +595,10 @@ async def test_error_file_id_for_failed_batch(): "litellm_call_id": "test-call-id", "api_base": "https://api.openai.com", "model_id": "test-model-id", - "model_name": "gpt-4o", + "model_name": "gpt-5.5", "response_cost": 0.0, "additional_headers": {}, - "litellm_model_name": "gpt-4o", + "litellm_model_name": "gpt-5.5", "unified_batch_id": "litellm_proxy;model_id:test-model-id;llm_batch_id:batch_abc123", } @@ -667,7 +667,7 @@ async def test_async_post_call_success_hook_twice_assert_no_unique_violation(): batch._hidden_params = { "model_id": "12345679", "response_cost": 0.0, - "litellm_model_name": "gpt-4o", + "litellm_model_name": "gpt-5.5", "unified_batch_id": "litellm_proxy;model_id:12345679;llm_batch_id:batch_685c5e5d63988190b85bdb2147ba131d", } @@ -1265,7 +1265,7 @@ async def test_completion_with_file_access_check(): ], } ], - "model": "gpt-4", + "model": "gpt-5.5", } # Should not raise exception @@ -1331,7 +1331,7 @@ async def test_responses_with_file_access_check(): }, } ], - "model": "gpt-4", + "model": "gpt-5.5", } # Should not raise exception @@ -1730,7 +1730,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_ await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=10, - target_model_names="gpt-4o,gpt-3.5", + target_model_names="gpt-5.5,gpt-3.5", ) assert str(exc_info.value) == ( @@ -1838,7 +1838,7 @@ async def test_return_unified_file_id_includes_expires_at(): create_file_request=create_file_request, internal_usage_cache=internal_usage_cache, litellm_parent_otel_span=None, - target_model_names_list=["gpt-4o"], + target_model_names_list=["gpt-5.5"], ) # Verify expires_at is passed through diff --git a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py index 52cb94ff34..c55b66b402 100644 --- a/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py +++ b/tests/enterprise/litellm_enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py @@ -107,10 +107,10 @@ async def test_new_project(prisma_client): description="Test project for unit testing", team_id=_team_id, metadata={"use_case_id": "TEST-001", "responsible_ai_id": "RAI-001"}, - models=["gpt-4", "gpt-3.5-turbo"], + models=["gpt-5.5", "gpt-5-mini"], max_budget=100.0, - model_rpm_limit={"gpt-4": 100}, - model_tpm_limit={"gpt-4": 1000}, + model_rpm_limit={"gpt-5.5": 100}, + model_tpm_limit={"gpt-5.5": 1000}, ) response = await new_project( @@ -130,12 +130,12 @@ async def test_new_project(prisma_client): assert response.project_alias == "test-project" assert response.description == "Test project for unit testing" assert response.team_id == _team_id - assert response.models == ["gpt-4", "gpt-3.5-turbo"] + assert response.models == ["gpt-5.5", "gpt-5-mini"] # model_rpm_limit and model_tpm_limit are stored in metadata assert response.metadata["use_case_id"] == "TEST-001" assert response.metadata["responsible_ai_id"] == "RAI-001" - assert response.metadata["model_rpm_limit"] == {"gpt-4": 100} - assert response.metadata["model_tpm_limit"] == {"gpt-4": 1000} + assert response.metadata["model_rpm_limit"] == {"gpt-5.5": 100} + assert response.metadata["model_tpm_limit"] == {"gpt-5.5": 1000} assert response.litellm_budget_table is not None assert response.litellm_budget_table.max_budget == 100.0 @@ -181,7 +181,7 @@ async def test_update_project(prisma_client): metadata={ "use_case_id": "TEST-002", }, - models=["gpt-4"], + models=["gpt-5.5"], max_budget=50.0, ) @@ -207,10 +207,10 @@ async def test_update_project(prisma_client): "use_case_id": "TEST-002-UPDATED", "additional_field": "new_value", }, - models=["gpt-4", "gpt-3.5-turbo", "claude-3"], + models=["gpt-5.5", "gpt-5-mini", "claude-3"], max_budget=200.0, - model_rpm_limit={"gpt-4": 200, "claude-3": 50}, - model_tpm_limit={"gpt-4": 2000, "claude-3": 500}, + model_rpm_limit={"gpt-5.5": 200, "claude-3": 50}, + model_tpm_limit={"gpt-5.5": 2000, "claude-3": 500}, ) update_response = await update_project( @@ -229,16 +229,16 @@ async def test_update_project(prisma_client): assert update_response.project_id == project_id assert update_response.project_alias == "test-project-updated" assert update_response.description == "Updated description" - assert update_response.models == ["gpt-4", "gpt-3.5-turbo", "claude-3"] + assert update_response.models == ["gpt-5.5", "gpt-5-mini", "claude-3"] # model_rpm_limit and model_tpm_limit are stored in metadata assert update_response.metadata["use_case_id"] == "TEST-002-UPDATED" assert update_response.metadata["additional_field"] == "new_value" assert update_response.metadata["model_rpm_limit"] == { - "gpt-4": 200, + "gpt-5.5": 200, "claude-3": 50, } assert update_response.metadata["model_tpm_limit"] == { - "gpt-4": 2000, + "gpt-5.5": 2000, "claude-3": 500, } assert update_response.litellm_budget_table is not None @@ -282,7 +282,7 @@ async def test_delete_project(prisma_client): project_data = NewProjectRequest( project_alias="test-project-delete", team_id=_team_id, - models=["gpt-4"], + models=["gpt-5.5"], max_budget=50.0, ) @@ -374,10 +374,10 @@ async def test_project_info(prisma_client): description="Test project info endpoint", team_id=_team_id, metadata={"use_case_id": "TEST-003", "cost_center": "engineering"}, - models=["gpt-4", "claude-3"], + models=["gpt-5.5", "claude-3"], max_budget=150.0, - model_rpm_limit={"gpt-4": 150}, - model_tpm_limit={"gpt-4": 1500}, + model_rpm_limit={"gpt-5.5": 150}, + model_tpm_limit={"gpt-5.5": 1500}, ) create_response = await new_project( @@ -410,12 +410,12 @@ async def test_project_info(prisma_client): assert info_response.project_alias == "test-project-info" assert info_response.description == "Test project info endpoint" assert info_response.team_id == _team_id - assert info_response.models == ["gpt-4", "claude-3"] + assert info_response.models == ["gpt-5.5", "claude-3"] # model_rpm_limit and model_tpm_limit are stored in metadata assert info_response.metadata["use_case_id"] == "TEST-003" assert info_response.metadata["cost_center"] == "engineering" - assert info_response.metadata["model_rpm_limit"] == {"gpt-4": 150} - assert info_response.metadata["model_tpm_limit"] == {"gpt-4": 1500} + assert info_response.metadata["model_rpm_limit"] == {"gpt-5.5": 150} + assert info_response.metadata["model_tpm_limit"] == {"gpt-5.5": 1500} assert info_response.litellm_budget_table is not None assert info_response.litellm_budget_table.max_budget == 150.0 @@ -439,12 +439,12 @@ def test_check_team_project_limits_models_not_in_team(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4", "gpt-3.5-turbo"], + models=["gpt-5.5", "gpt-5-mini"], ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4", "claude-3"], # claude-3 not in team + models=["gpt-5.5", "claude-3"], # claude-3 not in team ) with pytest.raises(Exception) as exc_info: @@ -465,13 +465,13 @@ def test_check_team_project_limits_budget_exceeds_team(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], max_budget=100.0, ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], max_budget=150.0, # exceeds team's 100.0 ) @@ -492,13 +492,13 @@ def test_check_team_project_limits_valid_subset(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4", "gpt-3.5-turbo", "claude-3"], + models=["gpt-5.5", "gpt-5-mini", "claude-3"], max_budget=1000.0, ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4", "gpt-3.5-turbo"], + models=["gpt-5.5", "gpt-5-mini"], max_budget=500.0, ) @@ -522,7 +522,7 @@ def test_check_team_project_limits_all_proxy_models(): data = NewProjectRequest( team_id="test-team", - models=["gpt-4", "claude-3", "anything-goes"], + models=["gpt-5.5", "claude-3", "anything-goes"], ) # Should not raise - team allows all models @@ -540,13 +540,13 @@ def test_check_team_project_limits_tpm_exceeds_team(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], tpm_limit=10000, ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], tpm_limit=20000, # exceeds team's 10000 ) @@ -567,12 +567,12 @@ def test_check_team_project_limits_negative_budget(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], max_budget=-10.0, ) @@ -593,12 +593,12 @@ def test_check_team_project_limits_soft_budget_gte_max(): team = LiteLLM_TeamTable( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], ) data = NewProjectRequest( team_id="test-team", - models=["gpt-4"], + models=["gpt-5.5"], max_budget=100.0, soft_budget=100.0, # equal to max, should fail ) diff --git a/tests/guardrails_tests/test_akto_guardrails.py b/tests/guardrails_tests/test_akto_guardrails.py index 83421f1313..901cdd3b95 100644 --- a/tests/guardrails_tests/test_akto_guardrails.py +++ b/tests/guardrails_tests/test_akto_guardrails.py @@ -62,7 +62,7 @@ def akto_ingest(): def sample_inputs() -> GenericGuardrailAPIInputs: return GenericGuardrailAPIInputs( texts=["Hello, how are you?"], - model="gpt-4", + model="gpt-5.5", ) @@ -200,7 +200,7 @@ def test_build_akto_payload_format(akto_validate, sample_inputs, sample_request_ req_wrapper = json.loads(payload["requestPayload"]) req_body = json.loads(req_wrapper["body"]) - assert req_body["model"] == "gpt-4" + assert req_body["model"] == "gpt-5.5" assert req_body["messages"][0]["content"] == "Hello, how are you?" tag = json.loads(payload["tag"]) @@ -486,7 +486,7 @@ async def test_fail_open_on_unreachable(): side_effect=httpx.ConnectError("Connection refused") ) - inputs = GenericGuardrailAPIInputs(texts=["test"], model="gpt-4") + inputs = GenericGuardrailAPIInputs(texts=["test"], model="gpt-5.5") result = await g.apply_guardrail( inputs=inputs, request_data={}, input_type="request" ) @@ -507,7 +507,7 @@ async def test_fail_closed_on_unreachable(): side_effect=httpx.ConnectError("Connection refused") ) - inputs = GenericGuardrailAPIInputs(texts=["test"], model="gpt-4") + inputs = GenericGuardrailAPIInputs(texts=["test"], model="gpt-5.5") with pytest.raises(HTTPException) as exc_info: await g.apply_guardrail(inputs=inputs, request_data={}, input_type="request") assert exc_info.value.status_code == 503 @@ -523,7 +523,7 @@ def test_fail_closed_generic_message(): ) with pytest.raises(HTTPException) as exc_info: g.handle_unreachable( - inputs=GenericGuardrailAPIInputs(texts=["test"], model="gpt-4"), + inputs=GenericGuardrailAPIInputs(texts=["test"], model="gpt-5.5"), error=Exception("http://internal-host:9090/secret-path"), ) assert "internal-host" not in exc_info.value.detail diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 5435721620..6e78a8c428 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -25,7 +25,7 @@ async def test_bedrock_guardrails_pii_masking(): ) request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, {"role": "assistant", "content": "Hello, how can I help you today?"}, @@ -65,7 +65,7 @@ async def test_bedrock_guardrails_pii_masking_content_list(): ) request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ { "role": "user", @@ -120,7 +120,7 @@ async def test_bedrock_guardrails_block_messages_api(): ) request_data = { - "model": "claude-3-5-sonnet-20240620", + "model": "claude-sonnet-4-5-20250929", "messages": [ { "role": "user", @@ -220,7 +220,7 @@ async def test_bedrock_guardrails_with_streaming(): litellm.callbacks.append(guardrail) request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Hi I like coffee"}], "stream": True, "metadata": {"guardrails": ["bedrock-post-guard"]}, @@ -264,7 +264,7 @@ async def test_bedrock_guardrails_with_streaming_no_violation(): litellm.callbacks.append(guardrail) request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "hi"}], "stream": True, "metadata": {"guardrails": ["bedrock-post-guard"]}, @@ -318,7 +318,7 @@ async def test_bedrock_guardrails_streaming_request_body_mock(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion", ) @@ -333,7 +333,7 @@ async def test_bedrock_guardrails_streaming_request_body_mock(): # Test data - simulating request data and assembled response request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "what's the capital of spain?"}], "stream": True, "metadata": {"guardrails": ["bedrock-post-guard"]}, @@ -396,7 +396,7 @@ async def test_bedrock_guardrail_aws_param_persistence(): ) as mock_get_creds: for i in range(3): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": f"request {i}"}], "stream": False, "metadata": {"guardrails": ["bedrock-post-guard"]}, @@ -583,7 +583,7 @@ async def test_bedrock_guardrail_masking_with_anonymized_response(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, ], @@ -657,7 +657,7 @@ async def test_bedrock_guardrail_uses_masked_output_without_masking_flags(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ { "role": "user", @@ -747,12 +747,12 @@ async def test_bedrock_guardrail_response_pii_masking_non_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion", ) request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "What's your credit card and phone number?"}, ], @@ -834,7 +834,7 @@ async def test_bedrock_guardrail_response_pii_masking_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ModelResponseStream( @@ -849,7 +849,7 @@ async def test_bedrock_guardrail_response_pii_masking_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ModelResponseStream( @@ -862,7 +862,7 @@ async def test_bedrock_guardrail_response_pii_masking_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ] @@ -870,7 +870,7 @@ async def test_bedrock_guardrail_response_pii_masking_streaming(): yield chunk request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "What's your email and SSN?"}, ], @@ -1001,7 +1001,7 @@ async def test_convert_to_bedrock_format_output_source(): ), ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion", ) @@ -1055,7 +1055,7 @@ async def test_convert_to_bedrock_format_post_call_streaming_hook(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ModelResponseStream( @@ -1068,7 +1068,7 @@ async def test_convert_to_bedrock_format_post_call_streaming_hook(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ] @@ -1097,7 +1097,7 @@ async def test_convert_to_bedrock_format_post_call_streaming_hook(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "What's your email?"}], "stream": True, } @@ -1223,7 +1223,7 @@ async def test_bedrock_guardrail_blocked_action_shows_output_text(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Tell me how to make explosives"}, ], @@ -1294,7 +1294,7 @@ async def test_bedrock_guardrail_blocked_action_empty_outputs(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Violent content here"}, ], @@ -1362,7 +1362,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_non_streaming(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Tell me how to make explosives"}, ], @@ -1442,7 +1442,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ModelResponseStream( @@ -1455,7 +1455,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion.chunk", ), ] @@ -1480,7 +1480,7 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): } request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Tell me how to make explosives"}], "stream": True, } @@ -1590,12 +1590,12 @@ async def test_bedrock_guardrail_post_call_success_hook_no_output_text(): ) ], created=1234567890, - model="gpt-4o", + model="gpt-5.5", object="chat.completion", ) data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Hello"}, ], diff --git a/tests/guardrails_tests/test_dynamoai_guardrails.py b/tests/guardrails_tests/test_dynamoai_guardrails.py index 8034f94a05..98f676a71d 100644 --- a/tests/guardrails_tests/test_dynamoai_guardrails.py +++ b/tests/guardrails_tests/test_dynamoai_guardrails.py @@ -53,7 +53,7 @@ async def test_dynamoai_blocks_content_with_block_action(): guardrail.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "This is harmful content"}], } @@ -102,7 +102,7 @@ async def test_dynamoai_allows_content_with_none_action(): guardrail.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Hello, how are you?"}], } diff --git a/tests/guardrails_tests/test_guardrail_load_balancing.py b/tests/guardrails_tests/test_guardrail_load_balancing.py index cf45b90673..4f71f83c43 100644 --- a/tests/guardrails_tests/test_guardrail_load_balancing.py +++ b/tests/guardrails_tests/test_guardrail_load_balancing.py @@ -65,8 +65,8 @@ async def test_proxy_logging_pre_call_hook_load_balancing(): router = Router( model_list=[ { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4", "api_key": "fake-key"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5", "api_key": "fake-key"}, } ], guardrail_list=guardrail_list, diff --git a/tests/guardrails_tests/test_guardrails_config.py b/tests/guardrails_tests/test_guardrails_config.py index 80b408a2dd..aaacb60726 100644 --- a/tests/guardrails_tests/test_guardrails_config.py +++ b/tests/guardrails_tests/test_guardrails_config.py @@ -58,7 +58,7 @@ def test_guardrail_masking_logging_only(): litellm.callbacks = [callback] messages = [{"role": "user", "content": "Hey, my name is Peter."}] response = completion( - model="gpt-3.5-turbo", messages=messages, mock_response="Hi Peter!" + model="gpt-5-mini", messages=messages, mock_response="Hi Peter!" ) assert response.choices[0].message.content == "Hi Peter!" # type: ignore @@ -82,7 +82,7 @@ def test_guardrail_list_of_event_hooks(): guardrail_name="custom-guard", event_hook=["pre_call", "post_call"] ) - data = {"model": "gpt-3.5-turbo", "metadata": {"guardrails": ["custom-guard"]}} + data = {"model": "gpt-5-mini", "metadata": {"guardrails": ["custom-guard"]}} assert cg.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call) assert cg.should_run_guardrail(data=data, event_type=GuardrailEventHooks.post_call) diff --git a/tests/guardrails_tests/test_lakera_v2.py b/tests/guardrails_tests/test_lakera_v2.py index b0134771ef..74e1935019 100644 --- a/tests/guardrails_tests/test_lakera_v2.py +++ b/tests/guardrails_tests/test_lakera_v2.py @@ -63,7 +63,7 @@ async def test_lakera_pre_call_hook_for_pii_masking(): "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com. My phone number is 555-123-4567", }, ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -170,7 +170,7 @@ async def test_lakera_blocks_non_pii_violations(): "content": "Some harmful content that triggers violations", } ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -236,7 +236,7 @@ async def test_lakera_only_pii_violations_are_masked(): data = { "messages": [{"role": "user", "content": "My email test@example.com here"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -423,7 +423,7 @@ async def test_lakera_blocks_flagged_content_with_user_scenario(): "content": "Some harmful content that should be blocked", } ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -487,7 +487,7 @@ async def test_lakera_monitor_mode_allows_flagged_content(): data = { "messages": [{"role": "user", "content": "Some harmful content"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -535,7 +535,7 @@ async def test_lakera_block_mode_raises_exception(): data = { "messages": [{"role": "user", "content": "Harmful content"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -578,7 +578,7 @@ async def test_lakera_monitor_mode_during_call(): data = { "messages": [{"role": "user", "content": "Test content"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -623,7 +623,7 @@ async def test_lakera_post_call_blocks_flagged_content(): data = { "messages": [{"role": "user", "content": "Harmful content"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -663,7 +663,7 @@ async def test_lakera_post_call_allows_clean_content(): data = { "messages": [{"role": "user", "content": "Hello"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } @@ -713,7 +713,7 @@ async def test_lakera_post_call_masks_pii_and_allows(): data = { "messages": [{"role": "user", "content": "Hello"}], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "metadata": {}, } diff --git a/tests/guardrails_tests/test_presidio_pii.py b/tests/guardrails_tests/test_presidio_pii.py index eda0c7bb5b..edc63bd941 100644 --- a/tests/guardrails_tests/test_presidio_pii.py +++ b/tests/guardrails_tests/test_presidio_pii.py @@ -153,7 +153,7 @@ async def test_presidio_pre_call_hook_with_blocked_entities(): "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com.", }, ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", } # Mock objects needed for the pre-call hook @@ -201,7 +201,7 @@ async def test_presidio_pre_call_hook_with_different_call_types(call_type): "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com. My phone number is 555-123-4567", }, ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", } # Mock objects needed for the pre-call hook @@ -286,7 +286,7 @@ async def test_output_parsing(): ] response = mock_completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=filtered_message, mock_response="Hello ! How can I assist you today?", ) diff --git a/tests/guardrails_tests/test_tracing_guardrails.py b/tests/guardrails_tests/test_tracing_guardrails.py index 841fe313b1..46f4f3e6e9 100644 --- a/tests/guardrails_tests/test_tracing_guardrails.py +++ b/tests/guardrails_tests/test_tracing_guardrails.py @@ -122,7 +122,7 @@ async def test_standard_logging_payload_includes_guardrail_information(): # 1. call the pre call hook with guardrail request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, ], @@ -221,7 +221,7 @@ async def test_langfuse_trace_includes_guardrail_information(): ) # 1. call the pre call hook with guardrail request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [ { "role": "user", @@ -343,7 +343,7 @@ async def test_bedrock_guardrail_status_blocked(): bedrock_guard.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "harmful content"}], "mock_response": "Hello", "metadata": {}, @@ -440,7 +440,7 @@ async def test_bedrock_guardrail_status_success(): bedrock_guard.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "safe content"}], "mock_response": "Hello", "metadata": {}, @@ -524,7 +524,7 @@ async def test_bedrock_guardrail_status_failure(): AsyncMock(side_effect=httpx.ConnectError("Connection failed")), ): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "test content"}], "mock_response": "Hello", "metadata": {}, @@ -615,7 +615,7 @@ async def test_noma_guardrail_status_blocked(): noma_guard.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "harmful content"}], "mock_response": "Hello", "metadata": {}, @@ -703,7 +703,7 @@ async def test_noma_guardrail_status_success(): noma_guard.async_handler, "post", AsyncMock(return_value=mock_response) ): request_data = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "safe content"}], "mock_response": "Hello", "metadata": {}, diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 8a6bed61a3..de6f7c38fe 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -99,7 +99,7 @@ async def test_azure_img_gen_health_check(): for attempt in range(max_retries): response = await litellm.ahealth_check( model_params={ - "model": "azure/dall-e-3", + "model": "azure/gpt-image-1", "api_base": os.getenv("AZURE_AI_API_BASE"), "api_key": os.getenv("AZURE_AI_API_KEY"), }, @@ -256,9 +256,9 @@ def test_update_litellm_params_for_health_check(): from litellm.proxy.health_check import _update_litellm_params_for_health_check # Test with health_check_model - model_info = {"health_check_model": "gpt-3.5-turbo"} + model_info = {"health_check_model": "gpt-5-mini"} litellm_params = { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake_key", } @@ -266,12 +266,12 @@ def test_update_litellm_params_for_health_check(): assert "messages" in updated_params assert isinstance(updated_params["messages"], list) - assert updated_params["model"] == "gpt-3.5-turbo" + assert updated_params["model"] == "gpt-5-mini" # Test without health_check_model model_info = {} litellm_params = { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake_key", } @@ -279,12 +279,12 @@ def test_update_litellm_params_for_health_check(): assert "messages" in updated_params assert isinstance(updated_params["messages"], list) - assert updated_params["model"] == "gpt-4" + assert updated_params["model"] == "gpt-5.5" # Test with health_check_voice for audio_speech mode model_info = {"mode": "audio_speech", "health_check_voice": "en-US-JennyNeural"} litellm_params = { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) @@ -294,7 +294,7 @@ def test_update_litellm_params_for_health_check(): # Test without health_check_voice for audio_speech mode model_info = {"mode": "audio_speech"} litellm_params = { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) @@ -304,7 +304,7 @@ def test_update_litellm_params_for_health_check(): # Test with health_check_voice for non-audio_speech mode model_info = {"mode": "chat", "health_check_voice": "en-US-JennyNeural"} litellm_params = { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) @@ -339,11 +339,11 @@ def test_update_litellm_params_for_health_check(): # Test that non-Bedrock models are not affected by Bedrock-specific logic litellm_params = { - "model": "openai/gpt-4", + "model": "openai/gpt-5.5", "api_key": "fake_key", } updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) - assert updated_params["model"] == "openai/gpt-4" # Should remain unchanged + assert updated_params["model"] == "openai/gpt-5.5" # Should remain unchanged # Test ALL cross-region inference profile prefixes (CRIS) cris_prefixes = ["us.", "eu.", "apac.", "jp.", "au.", "us-gov.", "global."] @@ -458,14 +458,14 @@ async def test_perform_health_check_filters_by_model_id(): # Two deployments with same model_name but different ids model_list = [ { - "model_name": "gpt-4", + "model_name": "gpt-5.5", "model_info": {"id": "deployment-id-1"}, - "litellm_params": {"model": "gpt-4", "api_key": "fake-key-1"}, + "litellm_params": {"model": "gpt-5.5", "api_key": "fake-key-1"}, }, { - "model_name": "gpt-4", + "model_name": "gpt-5.5", "model_info": {"id": "deployment-id-2"}, - "litellm_params": {"model": "gpt-4", "api_key": "fake-key-2"}, + "litellm_params": {"model": "gpt-5.5", "api_key": "fake-key-2"}, }, ] @@ -474,7 +474,7 @@ async def test_perform_health_check_filters_by_model_id(): async def mock_perform_health_check(m_list, details=True, **kwargs): captured_list.append(m_list) return ( - [{"model": "gpt-4", "api_key": m_list[0]["litellm_params"]["api_key"]}], + [{"model": "gpt-5.5", "api_key": m_list[0]["litellm_params"]["api_key"]}], [], {}, ) @@ -549,7 +549,7 @@ async def test_perform_health_check_with_health_check_model(): "litellm_params": {"model": "openai/*", "api_key": "fake-key"}, "model_info": { "mode": "chat", - "health_check_model": "openai/gpt-4o-mini", # Override model for health check + "health_check_model": "openai/gpt-5-mini", # Override model for health check }, } ] @@ -568,10 +568,10 @@ async def test_perform_health_check_with_health_check_model(): print("health check calls: ", health_check_calls) # Verify the health check used the override model - assert health_check_calls[0] == "openai/gpt-4o-mini" + assert health_check_calls[0] == "openai/gpt-5-mini" # Verify the result still shows the original model print("healthy endpoints: ", healthy_endpoints) - assert healthy_endpoints[0]["model"] == "openai/gpt-4o-mini" + assert healthy_endpoints[0]["model"] == "openai/gpt-5-mini" assert len(healthy_endpoints) == 1 assert len(unhealthy_endpoints) == 0 @@ -768,7 +768,7 @@ async def test_image_generation_health_check_prompt(monkeypatch): model_list = [ { - "litellm_params": {"model": "dall-e-3", "api_key": "fake-key"}, + "litellm_params": {"model": "gpt-image-1", "api_key": "fake-key"}, "model_info": { "mode": "image_generation", }, diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index e4dbe5f9f3..d64633413a 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -385,14 +385,14 @@ def test_get_valid_models_with_custom_llm_provider(custom_llm_provider): def test_bad_key(): key = "bad-key" - response = check_valid_key(model="gpt-3.5-turbo", api_key=key) + response = check_valid_key(model="gpt-5-mini", api_key=key) print(response, key) assert response == False def test_good_key(): key = os.environ["OPENAI_API_KEY"] - response = check_valid_key(model="gpt-3.5-turbo", api_key=key) + response = check_valid_key(model="gpt-5-mini", api_key=key) assert response == True @@ -406,7 +406,7 @@ def test_validate_environment_empty_model(): def test_validate_environment_api_key(): - response_obj = validate_environment(model="gpt-3.5-turbo", api_key="sk-my-test-key") + response_obj = validate_environment(model="gpt-5-mini", api_key="sk-my-test-key") assert ( response_obj["keys_in_environment"] is True ), f"Missing keys={response_obj['missing_keys']}" @@ -598,7 +598,7 @@ def test_get_chat_completion_prompt(): from litellm.litellm_core_utils.litellm_logging import Logging litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -610,7 +610,7 @@ def test_get_chat_completion_prompt(): updated_message = "hello world" litellm_logging_obj.get_chat_completion_prompt( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": updated_message}], non_default_params={}, prompt_id="1234", @@ -649,7 +649,7 @@ def test_redact_msgs_from_logs(): ) litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -700,14 +700,14 @@ def test_redact_embedding_response(): ] response_obj = litellm.EmbeddingResponse( - model="text-embedding-ada-002", + model="text-embedding-3-small", data=original_data, usage=original_usage, object="list", ) litellm_logging_obj = Logging( - model="text-embedding-ada-002", + model="text-embedding-3-small", messages=[{"role": "user", "content": "test input"}], stream=False, call_type="embedding", @@ -724,13 +724,13 @@ def test_redact_embedding_response(): # Assert the original response_obj is NOT modified assert response_obj.data == original_data assert response_obj.usage == original_usage - assert response_obj.model == "text-embedding-ada-002" + assert response_obj.model == "text-embedding-3-small" assert response_obj.object == "list" # Assert the redacted response preserves critical metadata assert _redacted_response_obj.usage == original_usage # usage should be preserved assert ( - _redacted_response_obj.model == "text-embedding-ada-002" + _redacted_response_obj.model == "text-embedding-3-small" ) # model should be preserved assert _redacted_response_obj.object == "list" # object should be preserved @@ -775,7 +775,7 @@ def test_redact_msgs_from_logs_with_dynamic_params(): ) litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -934,7 +934,7 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): litellm.success_callback = ["langfuse"] litellm_call_id = "my-unique-call-id" litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -951,7 +951,7 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): metadata["existing_trace_id"] = langfuse_existing_trace_id litellm.completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hey how's it going?"}], mock_response="Hey!", litellm_logging_obj=litellm_logging_obj, @@ -1633,7 +1633,7 @@ def test_get_valid_models_openai_proxy(monkeypatch): "object": "list", "data": [ { - "id": "gpt-4o", + "id": "gpt-5.5", "object": "model", "created": 1686935002, "owned_by": "organization-owner", @@ -1650,7 +1650,7 @@ def test_get_valid_models_openai_proxy(monkeypatch): litellm.module_level_client, "get", return_value=mock_response ) as mock_post: valid_models = get_valid_models(check_provider_endpoint=True) - assert "litellm_proxy/gpt-4o" in valid_models + assert "litellm_proxy/gpt-5.5" in valid_models def test_get_valid_models_fireworks_ai(monkeypatch): @@ -1807,7 +1807,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e(monkeypatch): curr_len_failure_callback = len(litellm.failure_callback) litellm.completion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing langfuse", ) @@ -1922,7 +1922,7 @@ async def test_add_custom_logger_callback_to_specific_event_with_duplicates( # Make a completion call await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing duplicate callbacks", ) @@ -1961,7 +1961,7 @@ async def test_add_custom_logger_callback_to_specific_event_with_duplicates_succ # Make a completion call await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing duplicate callbacks", ) @@ -1996,7 +1996,7 @@ async def test_add_custom_logger_callback_to_specific_event_with_duplicates_call # Make a completion call await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing duplicate callbacks", ) @@ -2011,7 +2011,7 @@ async def test_add_custom_logger_callback_to_specific_event_with_duplicates_call for _ in range(10): await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing duplicate callbacks", ) @@ -2040,7 +2040,7 @@ def test_add_custom_logger_callback_to_specific_event_e2e_failure(monkeypatch): curr_len_failure_callback = len(litellm.failure_callback) litellm.completion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Testing langfuse", ) @@ -2069,7 +2069,7 @@ async def test_wrapper_kwargs_passthrough(): return await mock_original(**kwargs) # Test kwargs - test_kwargs = {"base_model": "gpt-4o-mini"} + test_kwargs = {"base_model": "gpt-5-mini"} # Call decorated function await test_function(**test_kwargs) @@ -2089,7 +2089,7 @@ async def test_wrapper_kwargs_passthrough(): # get base model assert ( litellm_logging_obj.model_call_details["litellm_params"]["base_model"] - == "gpt-4o-mini" + == "gpt-5-mini" ) @@ -2327,15 +2327,15 @@ def test_get_valid_models_from_provider(): valid_models = get_valid_models(custom_llm_provider="openai") assert len(valid_models) > 0 - assert "gpt-4o-mini" in valid_models + assert "gpt-5-mini" in valid_models print("Valid models: ", valid_models) - valid_models.remove("gpt-4o-mini") - assert "gpt-4o-mini" not in valid_models + valid_models.remove("gpt-5-mini") + assert "gpt-5-mini" not in valid_models valid_models = get_valid_models(custom_llm_provider="openai") assert len(valid_models) > 0 - assert "gpt-4o-mini" in valid_models + assert "gpt-5-mini" in valid_models def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): @@ -2347,7 +2347,7 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "123") _model_cache.set_cached_model_info( - "openai", litellm_params=None, available_models=["gpt-4o-mini"] + "openai", litellm_params=None, available_models=["gpt-5-mini"] ) monkeypatch.delenv("OPENAI_API_KEY") @@ -2471,10 +2471,10 @@ def test_get_base_model_from_metadata(): # Test 1: base_model in metadata (Chat Completions API pattern) model_call_details_with_metadata = { - "litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-4"}}} + "litellm_params": {"metadata": {"model_info": {"base_model": "azure/gpt-5.5"}}} } result = _get_base_model_from_metadata(model_call_details_with_metadata) - assert result == "azure/gpt-4", f"Expected 'azure/gpt-4', got {result}" + assert result == "azure/gpt-5.5", f"Expected 'azure/gpt-5.5', got {result}" # Test 2: base_model in litellm_metadata (Responses API and generic API calls pattern) model_call_details_with_litellm_metadata = { @@ -2487,12 +2487,12 @@ def test_get_base_model_from_metadata(): # Test 3: base_model in litellm_params (direct base_model) model_call_details_with_direct_base_model = { - "litellm_params": {"base_model": "azure/gpt-3.5-turbo"} + "litellm_params": {"base_model": "azure/gpt-5-mini"} } result = _get_base_model_from_metadata(model_call_details_with_direct_base_model) assert ( - result == "azure/gpt-3.5-turbo" - ), f"Expected 'azure/gpt-3.5-turbo', got {result}" + result == "azure/gpt-5-mini" + ), f"Expected 'azure/gpt-5-mini', got {result}" # Test 4: metadata takes precedence over litellm_metadata model_call_details_with_both = { diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 56a752be56..30f444b9ac 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -363,7 +363,7 @@ class BaseResponsesAPITest(ABC): litellm._turn_on_debug() response = await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="Tell me a three sentence bedtime story about a unicorn.", ) print("Initial response=", json.dumps(response, indent=4, default=str)) @@ -771,7 +771,7 @@ class BaseResponsesAPITest(ABC): except litellm.BadRequestError as e: if "shell" in str(e).lower() and "not supported" in str(e).lower(): pytest.skip( - "Shell tool is not supported for this model (e.g. gpt-4o); use a model that supports shell" + "Shell tool is not supported for this model (e.g. gpt-5.5); use a model that supports shell" ) raise validate_responses_api_response(response, final_chunk=True) @@ -785,7 +785,7 @@ class BaseResponsesAPITest(ABC): Calls aresponses(..., tools=[shell], stream=True), then iterates the stream and asserts at least one event is shell-related or response output contains shell_call. - Skips when model does not support shell (e.g. gpt-4o). + Skips when model does not support shell (e.g. gpt-5.5). """ base_completion_call_args = self.get_base_completion_call_args() model = ( diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 8f5278698b..e2c50810cc 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -72,7 +72,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = BaseResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -142,7 +142,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = BaseResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -188,7 +188,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = BaseResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, ) @@ -214,7 +214,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = BaseResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, ) @@ -240,7 +240,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = BaseResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, ) @@ -280,7 +280,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = ResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -355,7 +355,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = ResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -417,7 +417,7 @@ class TestBaseResponsesAPIStreamingIterator: # Create the iterator instance iterator = SyncResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -475,7 +475,7 @@ class TestBaseResponsesAPIStreamingIterator: iterator = ResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, @@ -554,7 +554,7 @@ class TestBaseResponsesAPIStreamingIterator: iterator = ResponsesAPIStreamingIterator( response=mock_response, - model="gpt-4", + model="gpt-5.5", responses_api_provider_config=mock_config, logging_obj=mock_logging_obj, litellm_metadata={"model_info": {"id": "model_123"}}, diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 09cc5be739..ea8b8fa886 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -28,7 +28,7 @@ from base_responses_api import BaseResponsesAPITest, validate_responses_api_resp class TestOpenAIResponsesAPITest(BaseResponsesAPITest): def get_base_completion_call_args(self): return { - "model": "openai/gpt-4o", + "model": "openai/gpt-5.5", } def get_base_completion_reasoning_call_args(self): @@ -104,7 +104,7 @@ def test_basic_openai_responses_api_streaming_with_logging(): litellm.set_verbose = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] - request_model = "gpt-4o" + request_model = "gpt-5.5" response = litellm.responses( model=request_model, input="hi", @@ -176,7 +176,7 @@ async def test_basic_openai_responses_api_non_streaming_with_logging(): litellm.set_verbose = True test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] - request_model = "gpt-4o" + request_model = "gpt-5.5" response = await litellm.aresponses( model=request_model, input="hi", @@ -215,13 +215,13 @@ async def test_openai_responses_api_returns_headers(sync_mode): if sync_mode: response = litellm.responses( - model="gpt-4o", + model="gpt-5.5", input="Say hello", max_output_tokens=20, ) else: response = await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="Say hello", max_output_tokens=20, ) @@ -471,7 +471,7 @@ async def test_openai_responses_api_streaming_validation(sync_mode): if sync_mode: response = litellm.responses( - model="gpt-4o", + model="gpt-5.5", input="Tell me about artificial intelligence in 3 sentences.", stream=True, ) @@ -481,7 +481,7 @@ async def test_openai_responses_api_streaming_validation(sync_mode): event_types_seen.add(event.type) else: response = await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="Tell me about artificial intelligence in 3 sentences.", stream=True, ) @@ -511,7 +511,7 @@ async def test_openai_responses_litellm_router(sync_mode): { "model_name": "gpt4o-special-alias", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": os.getenv("OPENAI_API_KEY"), }, } @@ -556,7 +556,7 @@ async def test_openai_responses_litellm_router_streaming(sync_mode): { "model_name": "gpt4o-special-alias", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": os.getenv("OPENAI_API_KEY"), }, } @@ -605,7 +605,7 @@ async def test_openai_responses_litellm_router_no_metadata(): "object": "response", "created_at": 1741476542, "status": "completed", - "model": "gpt-4o", + "model": "gpt-5.5", "output": [ { "type": "message", @@ -664,7 +664,7 @@ async def test_openai_responses_litellm_router_no_metadata(): { "model_name": "gpt4o-special-alias", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": "fake-key", }, } @@ -704,7 +704,7 @@ async def test_openai_responses_litellm_router_with_metadata(): "object": "response", "created_at": 1741476542, "status": "completed", - "model": "gpt-4o", + "model": "gpt-5.5", "output": [ { "type": "message", @@ -762,7 +762,7 @@ async def test_openai_responses_litellm_router_with_metadata(): { "model_name": "gpt4o-special-alias", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": "fake-key", }, } @@ -802,7 +802,7 @@ async def test_openai_responses_litellm_router_with_prompt(): "object": "response", "created_at": 1741476542, "status": "completed", - "model": "gpt-4o", + "model": "gpt-5.5", "output": [], "parallel_tool_calls": True, "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, @@ -844,7 +844,7 @@ async def test_openai_responses_litellm_router_with_prompt(): { "model_name": "gpt4o-special-alias", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": "fake-key", }, } @@ -865,7 +865,7 @@ async def test_openai_responses_litellm_router_with_prompt(): def test_bad_request_bad_param_error(): """Raise a BadRequestError when an invalid parameter value is provided""" try: - litellm.responses(model="gpt-4o", input="This should fail", temperature=2000) + litellm.responses(model="gpt-5.5", input="This should fail", temperature=2000) pytest.fail("Expected BadRequestError but no exception was raised") except litellm.BadRequestError as e: print(f"Exception raised: {e}") @@ -881,7 +881,7 @@ async def test_async_bad_request_bad_param_error(): """Raise a BadRequestError when an invalid parameter value is provided""" try: await litellm.aresponses( - model="gpt-4o", input="This should fail", temperature=2000 + model="gpt-5.5", input="This should fail", temperature=2000 ) pytest.fail("Expected BadRequestError but no exception was raised") except litellm.BadRequestError as e: @@ -1280,7 +1280,7 @@ async def test_openai_responses_api_field_types(): # Test with store=True response = await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="hi", ) @@ -1292,7 +1292,7 @@ async def test_openai_responses_api_field_types(): assert response.store is True, "store field should match input value" # Test without store parameter - response_without_store = await litellm.aresponses(model="gpt-4o", input="hi") + response_without_store = await litellm.aresponses(model="gpt-5.5", input="hi") # Verify created_at is still an integer assert isinstance( @@ -1310,7 +1310,7 @@ async def test_store_field_transformation(): # Initialize logging object with required parameters logging_obj = LiteLLMLoggingObj( - model="gpt-4o", + model="gpt-5.5", messages=[], stream=False, call_type="aresponses", @@ -1323,7 +1323,7 @@ async def test_store_field_transformation(): base_response = { "id": "test_id", "created_at": 1751443898, - "model": "gpt-4o", + "model": "gpt-5.5", "object": "response", "output": [ { @@ -1378,7 +1378,7 @@ async def test_store_field_transformation(): # Test when store=True in request logging_obj.optional_params = {"store": True} response = config.transform_response_api_response( - model="gpt-4o", raw_response=mock_response_store_true, logging_obj=logging_obj + model="gpt-5.5", raw_response=mock_response_store_true, logging_obj=logging_obj ) assert ( response.store is True @@ -1387,7 +1387,7 @@ async def test_store_field_transformation(): # Test when store=False in request logging_obj.optional_params = {"store": False} response = config.transform_response_api_response( - model="gpt-4o", raw_response=mock_response_store_false, logging_obj=logging_obj + model="gpt-5.5", raw_response=mock_response_store_false, logging_obj=logging_obj ) assert ( response.store is False @@ -1395,7 +1395,7 @@ async def test_store_field_transformation(): # Test when store not in request but API returns null response = config.transform_response_api_response( - model="gpt-4o", raw_response=mock_response_store_null, logging_obj=logging_obj + model="gpt-5.5", raw_response=mock_response_store_null, logging_obj=logging_obj ) assert ( response.store is None @@ -1403,7 +1403,7 @@ async def test_store_field_transformation(): # Test when store not in request and API omits store field response = config.transform_response_api_response( - model="gpt-4o", raw_response=mock_response_no_store, logging_obj=logging_obj + model="gpt-5.5", raw_response=mock_response_no_store, logging_obj=logging_obj ) assert ( response.store is None @@ -1484,7 +1484,7 @@ async def test_aresponses_service_tier_and_safety_identifier(): # Call aresponses with service_tier and safety_identifier response = await litellm.aresponses( - model="openai/gpt-4o", + model="openai/gpt-5.5", input="Test with service tier and safety identifier", service_tier="flex", safety_identifier="123", @@ -1502,7 +1502,7 @@ async def test_aresponses_service_tier_and_safety_identifier(): assert ( request_body["safety_identifier"] == "123" ), "safety_identifier should be '123' in request body" - assert request_body["model"] == "gpt-4o" + assert request_body["model"] == "gpt-5.5" assert request_body["input"] == "Test with service tier and safety identifier" # Validate the response @@ -1609,7 +1609,7 @@ async def test_openai_gpt5_reasoning_effort_parameter(): @pytest.mark.parametrize("stream", [True, False]) async def test_basic_openai_responses_with_websearch(stream): litellm._turn_on_debug() - request_model = "gpt-4o" + request_model = "gpt-5.5" response = await litellm.aresponses( model=request_model, stream=stream, @@ -1715,7 +1715,7 @@ def extra_body_mock_response_data(): "object": "response", "created_at": 1234567890, "status": "completed", - "model": "gpt-4o", + "model": "gpt-5.5", "output": [ { "type": "message", @@ -1747,7 +1747,7 @@ async def test_aresponses_extra_body_params_passed(extra_body_mock_response_data mock_post.return_value = MockResponse(extra_body_mock_response_data, 200) response = await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="Test input", max_output_tokens=20, extra_body={ @@ -1768,7 +1768,7 @@ async def test_aresponses_extra_body_params_passed(extra_body_mock_response_data assert request_body["custom_param_2"]["nested"] == "value2" assert "experimental_feature" in request_body assert request_body["experimental_feature"] is True - assert request_body["model"] == "gpt-4o" + assert request_body["model"] == "gpt-5.5" assert request_body["input"] == "Test input" @@ -1779,7 +1779,7 @@ def test_responses_extra_body_params_passed_sync(extra_body_mock_response_data): return_value=MockResponse(extra_body_mock_response_data, 200), ) as mock_post: response = litellm.responses( - model="gpt-4o", + model="gpt-5.5", input="Sync test", max_output_tokens=20, extra_body={ @@ -1797,7 +1797,7 @@ def test_responses_extra_body_params_passed_sync(extra_body_mock_response_data): assert request_body["sync_custom_param"] == "sync_value" assert "another_param" in request_body assert request_body["another_param"] == 42 - assert request_body["model"] == "gpt-4o" + assert request_body["model"] == "gpt-5.5" @pytest.mark.asyncio @@ -1810,7 +1810,7 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data mock_post.return_value = MockResponse(extra_body_mock_response_data, 200) await litellm.aresponses( - model="gpt-4o", + model="gpt-5.5", input="Test", temperature=0.7, max_output_tokens=20, @@ -1847,13 +1847,13 @@ async def test_openai_compact_responses_api(sync_mode): try: if sync_mode: response = litellm.compact_responses( - model="openai/gpt-4o", + model="openai/gpt-5.5", input=input_messages, instructions="Be helpful and concise", ) else: response = await litellm.acompact_responses( - model="openai/gpt-4o", + model="openai/gpt-5.5", input=input_messages, instructions="Be helpful and concise", ) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 371b27c5b2..7a478e494b 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -1379,7 +1379,7 @@ def test_anthropic_mcp_server_tool_use(spec: str): ] params = { - "model": "anthropic/claude-sonnet-4-20250514", + "model": "anthropic/claude-sonnet-4-5-20250929", "messages": [{"role": "user", "content": "Who won the World Cup in 2022?"}], "tools": tools, } @@ -1392,7 +1392,7 @@ def test_anthropic_mcp_server_tool_use(spec: str): @pytest.mark.parametrize( - "model", ["openai/gpt-4.1", "anthropic/claude-sonnet-4-20250514"] + "model", ["openai/gpt-4.1", "anthropic/claude-sonnet-4-5-20250929"] ) @pytest.mark.skipif( os.getenv("ZAPIER_CI_CD_MCP_TOKEN") is None, reason="ZAPIER_CI_CD_MCP_TOKEN not set" @@ -1506,8 +1506,8 @@ def test_anthropic_tool_cache_control(): } ] - vertex_ai_model = "vertex_ai/claude-sonnet-4@20250514" - anthropic_api_model = "claude-sonnet-4-20250514" + vertex_ai_model = "vertex_ai/claude-sonnet-4-5@20250929" + anthropic_api_model = "claude-sonnet-4-5-20250929" result = return_raw_request( endpoint=CallTypes.completion, kwargs={ diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index b40ce11bb9..93acf01683 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -2037,7 +2037,7 @@ def test_drop_store_param_for_anthropic(): Ref: https://github.com/BerriAI/litellm/issues/19700 """ optional_params = get_optional_params( - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-5-20250929", custom_llm_provider="anthropic", drop_params=True, store=True, @@ -2053,7 +2053,7 @@ def test_additional_drop_params_store_for_anthropic(): Ref: https://github.com/BerriAI/litellm/issues/19700 """ optional_params = get_optional_params( - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-5-20250929", custom_llm_provider="anthropic", additional_drop_params=["store"], store=True, diff --git a/tests/local_testing/create_mock_standard_logging_payload.py b/tests/local_testing/create_mock_standard_logging_payload.py index 2fd6a4ffa8..106328e95e 100644 --- a/tests/local_testing/create_mock_standard_logging_payload.py +++ b/tests/local_testing/create_mock_standard_logging_payload.py @@ -43,9 +43,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", @@ -94,9 +94,9 @@ def create_standard_logging_payload_with_long_content() -> StandardLoggingPayloa endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", diff --git a/tests/logging_callback_tests/create_mock_standard_logging_payload.py b/tests/logging_callback_tests/create_mock_standard_logging_payload.py index 2fd6a4ffa8..106328e95e 100644 --- a/tests/logging_callback_tests/create_mock_standard_logging_payload.py +++ b/tests/logging_callback_tests/create_mock_standard_logging_payload.py @@ -43,9 +43,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", @@ -94,9 +94,9 @@ def create_standard_logging_payload_with_long_content() -> StandardLoggingPayloa endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", diff --git a/tests/logging_callback_tests/test_alerting.py b/tests/logging_callback_tests/test_alerting.py index 134056de80..7cf88d49e2 100644 --- a/tests/logging_callback_tests/test_alerting.py +++ b/tests/logging_callback_tests/test_alerting.py @@ -43,7 +43,7 @@ from litellm.utils import get_api_base "model, optional_params, expected_api_base", [ ("openai/my-fake-model", {"api_base": "my-fake-api-base"}, "my-fake-api-base"), - ("gpt-3.5-turbo", {}, "https://api.openai.com"), + ("gpt-5-mini", {}, "https://api.openai.com"), ], ) def test_get_api_base_unit_test(model, optional_params, expected_api_base): @@ -254,7 +254,7 @@ async def test_daily_reports_unit_test(slack_alerting): model_list=[ { "model_name": "test-gpt", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "litellm_params": {"model": "gpt-5-mini"}, "model_info": {"id": "1234"}, } ] @@ -286,16 +286,16 @@ async def test_daily_reports_completion(slack_alerting): router = litellm.Router( model_list=[ { - "model_name": "gpt-5", + "model_name": "gpt-5.5", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", }, } ] ) await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) @@ -310,15 +310,15 @@ async def test_daily_reports_completion(slack_alerting): router = litellm.Router( model_list=[ { - "model_name": "gpt-5", - "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "bad_key"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5-mini", "api_key": "bad_key"}, } ] ) try: await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) except Exception as e: @@ -347,9 +347,9 @@ async def test_daily_reports_redis_cache_scheduler(): router = litellm.Router( model_list=[ { - "model_name": "gpt-5", + "model_name": "gpt-5.5", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", }, } ] @@ -388,16 +388,16 @@ async def test_send_llm_exception_to_slack(): router = litellm.Router( model_list=[ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "bad_key", }, }, { "model_name": "gpt-5-good", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", }, }, ], @@ -407,7 +407,7 @@ async def test_send_llm_exception_to_slack(): ) try: await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) except Exception: @@ -582,9 +582,9 @@ async def test_webhook_alerting(alerting_type): @pytest.mark.parametrize( "model, api_base, llm_provider, vertex_project, vertex_location", [ - ("gpt-3.5-turbo", None, "openai", None, None), + ("gpt-5-mini", None, "openai", None, None), ( - "azure/gpt-3.5-turbo", + "azure/gpt-5-mini", "https://openai-gpt-4-test-v-1.openai.azure.com", "azure", None, @@ -688,9 +688,9 @@ async def test_outage_alerting_called( @pytest.mark.parametrize( "model, api_base, llm_provider, vertex_project, vertex_location", [ - ("gpt-3.5-turbo", None, "openai", None, None), + ("gpt-5-mini", None, "openai", None, None), ( - "azure/gpt-3.5-turbo", + "azure/gpt-5-mini", "https://openai-gpt-4-test-v-1.openai.azure.com", "azure", None, @@ -800,7 +800,7 @@ async def test_langfuse_trace_id(): litellm.success_callback = ["langfuse"] litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -810,7 +810,7 @@ async def test_langfuse_trace_id(): ) litellm.completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hey how's it going?"}], mock_response="Hey!", litellm_logging_obj=litellm_logging_obj, diff --git a/tests/logging_callback_tests/test_amazing_s3_logs.py b/tests/logging_callback_tests/test_amazing_s3_logs.py index 59a8c4a8cf..dab2a0cc0b 100644 --- a/tests/logging_callback_tests/test_amazing_s3_logs.py +++ b/tests/logging_callback_tests/test_amazing_s3_logs.py @@ -36,7 +36,7 @@ async def test_basic_s3_logging(sync_mode, streaming): response_id = None if sync_mode is True: response = litellm.completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "This is a test"}], mock_response="It's simple to use and easy to get started", stream=streaming, @@ -50,7 +50,7 @@ async def test_basic_s3_logging(sync_mode, streaming): time.sleep(2) else: response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "This is a test"}], mock_response="It's simple to use and easy to get started", stream=streaming, @@ -102,7 +102,7 @@ async def test_basic_s3_v2_logging(streaming): litellm.set_verbose = True response_id = None response = await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", messages=[{"role": "user", "content": "This is a test"}], mock_response="It's simple to use and easy to get started", stream=streaming, @@ -149,7 +149,7 @@ async def test_basic_s3_v2_logging_failure(): # Mock the upload process but still make the httpx call url = f"https://test-bucket.s3.us-west-2.amazonaws.com/{batch_logging_element.s3_object_key}" headers = {"Content-Type": "application/json"} - data = '{"model": "gpt-4o-mini"}' + data = '{"model": "gpt-5-mini"}' # Make the actual httpx call we want to test await s3_v2_logger.async_httpx_client.put(url=url, headers=headers, data=data) @@ -169,7 +169,7 @@ async def test_basic_s3_v2_logging_failure(): # Trigger a failure by using invalid API key try: response = await litellm.acompletion( - model="gpt-4o-mini", + model="gpt-5-mini", api_key="invalid-api-key", messages=[{"role": "user", "content": "This is a test"}], ) @@ -203,7 +203,7 @@ async def test_basic_s3_v2_logging_failure(): # Verify JSON data was included data = call_args[1]["data"] assert data is not None - assert '"model": "gpt-4o-mini"' in data + assert '"model": "gpt-5-mini"' in data print("✓ S3 request data contains expected log payload") @@ -256,7 +256,7 @@ def test_s3_logging(): async def _test(): return await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": f"This is a test {curr_time}"}], max_tokens=10, temperature=0.7, @@ -269,7 +269,7 @@ def test_s3_logging(): async def _test(): return await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": f"This is a test {curr_time}"}], max_tokens=10, temperature=0.7, diff --git a/tests/logging_callback_tests/test_assemble_streaming_responses.py b/tests/logging_callback_tests/test_assemble_streaming_responses.py index 20e46db229..919b76e95a 100644 --- a/tests/logging_callback_tests/test_assemble_streaming_responses.py +++ b/tests/logging_callback_tests/test_assemble_streaming_responses.py @@ -65,7 +65,7 @@ def test_assemble_complete_response_from_streaming_chunks_1(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, @@ -105,7 +105,7 @@ def test_assemble_complete_response_from_streaming_chunks_1(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, @@ -166,7 +166,7 @@ def test_assemble_complete_response_from_streaming_chunks_2(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, @@ -208,7 +208,7 @@ def test_assemble_complete_response_from_streaming_chunks_2(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, @@ -263,7 +263,7 @@ def test_assemble_complete_response_from_streaming_chunks_3(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, @@ -340,7 +340,7 @@ def test_assemble_complete_response_from_streaming_chunks_4(is_async): ) ], "created": 1721353246, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "object": "chat.completion.chunk", "system_fingerprint": None, "usage": None, diff --git a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py index 3e8d59b299..d6d0652ed7 100644 --- a/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py +++ b/tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py @@ -445,7 +445,7 @@ async def test_openai_with_knowledge_base_mock_openai(setup_vector_store_registr mock_response.id = "chatcmpl-123" mock_response.object = "chat.completion" mock_response.created = 1234567890 - mock_response.model = "gpt-4" + mock_response.model = "gpt-5.5" # Store the request for verification captured_request.update(kwargs) @@ -459,7 +459,7 @@ async def test_openai_with_knowledge_base_mock_openai(setup_vector_store_registr try: await litellm.acompletion( - model="gpt-4", + model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], vector_store_ids=["T37J8R4WTM"], client=client, @@ -521,7 +521,7 @@ async def test_openai_with_vector_store_ids_in_tool_call_mock_openai( mock_response.id = "chatcmpl-123" mock_response.object = "chat.completion" mock_response.created = 1234567890 - mock_response.model = "gpt-4" + mock_response.model = "gpt-5.5" # Store the request for verification captured_request.update(kwargs) @@ -535,7 +535,7 @@ async def test_openai_with_vector_store_ids_in_tool_call_mock_openai( try: await litellm.acompletion( - model="gpt-4", + model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}], client=client, @@ -594,7 +594,7 @@ async def test_openai_with_mixed_tool_call_mock_openai(setup_vector_store_regist mock_response.id = "chatcmpl-123" mock_response.object = "chat.completion" mock_response.created = 1234567890 - mock_response.model = "gpt-4" + mock_response.model = "gpt-5.5" # Store the request for verification captured_request.update(kwargs) @@ -608,7 +608,7 @@ async def test_openai_with_mixed_tool_call_mock_openai(setup_vector_store_regist try: await litellm.acompletion( - model="gpt-4", + model="gpt-5.5", messages=[{"role": "user", "content": "what is litellm?"}], tools=[ {"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}, @@ -642,7 +642,7 @@ async def test_openai_with_mixed_tool_call_mock_openai(setup_vector_store_regist # test_custom_logger = MockCustomLogger() # litellm.set_verbose = True # await litellm.acompletion( -# model="gpt-4", +# model="gpt-5.5", # messages=[{"role": "user", "content": "what is litellm?"}], # vector_store_ids = [ # "T37J8R4WTM" @@ -834,7 +834,7 @@ async def test_provider_specific_fields_in_proxy_http_response( # Initialize proxy await initialize( - model="gpt-3.5-turbo", + model="gpt-5-mini", alias=None, api_base=None, debug=False, @@ -857,7 +857,7 @@ async def test_provider_specific_fields_in_proxy_http_response( # Create mock response with provider_specific_fields mock_response = litellm.ModelResponse( id="test-123", - model="gpt-3.5-turbo", + model="gpt-5-mini", created=1234567890, object="chat.completion", ) @@ -897,7 +897,7 @@ async def test_provider_specific_fields_in_proxy_http_response( response = client.post( "/v1/chat/completions", json={ - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "What is litellm?"}], }, ) diff --git a/tests/logging_callback_tests/test_custom_callback_router.py b/tests/logging_callback_tests/test_custom_callback_router.py index 63d8b14f48..70da10ffee 100644 --- a/tests/logging_callback_tests/test_custom_callback_router.py +++ b/tests/logging_callback_tests/test_custom_callback_router.py @@ -441,7 +441,7 @@ async def test_async_chat_azure(): # failure model_list = [ { - "model_name": "gpt-3.5-turbo", # openai model name + "model_name": "gpt-5-mini", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4o-new-test", "api_key": "my-bad-key", @@ -458,7 +458,7 @@ async def test_async_chat_azure(): router3 = Router(model_list=model_list, num_retries=0) # type: ignore try: response = await router3.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], ) print(f"response in router3 acompletion: {response}") @@ -547,7 +547,7 @@ async def test_async_chat_azure_with_fallbacks(): # with fallbacks model_list = [ { - "model_name": "gpt-3.5-turbo", # openai model name + "model_name": "gpt-5-mini", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", "api_key": "my-bad-key", @@ -568,13 +568,13 @@ async def test_async_chat_azure_with_fallbacks(): ] router = Router( model_list=model_list, - fallbacks=[{"gpt-3.5-turbo": ["gpt-3.5-turbo-16k"]}], + fallbacks=[{"gpt-5-mini": ["gpt-3.5-turbo-16k"]}], retry_policy=litellm.router.RetryPolicy( AuthenticationErrorRetries=0, ), ) # type: ignore response = await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], ) await asyncio.sleep(2) @@ -731,9 +731,9 @@ async def test_async_embedding_azure_caching(): router = Router( model_list=[ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "openai/text-embedding-ada-002", + "model": "openai/text-embedding-3-small", }, } ] @@ -741,13 +741,13 @@ async def test_async_embedding_azure_caching(): litellm.callbacks = [customHandler_caching] unique_time = time.time() response1 = await router.aembedding( - model="text-embedding-ada-002", + model="text-embedding-3-small", input=[f"good morning from litellm1 {unique_time}"], caching=True, ) await asyncio.sleep(1) # set cache is async for aembedding() response2 = await router.aembedding( - model="text-embedding-ada-002", + model="text-embedding-3-small", input=[f"good morning from litellm1 {unique_time}"], caching=True, ) @@ -776,7 +776,7 @@ async def test_rate_limit_error_callback(): { "model_name": "my-test-gpt", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "mock_response": "litellm.RateLimitError", }, } diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 71593b0ae8..bc7a9a211a 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -54,9 +54,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-4.1-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-4.1-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", @@ -195,7 +195,7 @@ async def test_datadog_logging_http_request(): # Make the completion call for _ in range(5): response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "what llm are u"}], max_tokens=10, temperature=0.2, @@ -279,7 +279,7 @@ async def test_datadog_logging_http_request(): # Check specific fields assert message["call_type"] == "acompletion" - assert message["model"] == "gpt-3.5-turbo" + assert message["model"] == "gpt-4.1-mini" assert isinstance(message["model_parameters"], dict) assert "temperature" in message["model_parameters"] assert "max_tokens" in message["model_parameters"] @@ -411,7 +411,7 @@ async def test_datadog_log_redis_failures(): # Make the completion call for _ in range(3): response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "what llm are u"}], max_tokens=10, temperature=0.2, @@ -469,7 +469,7 @@ async def test_datadog_logging(): litellm.success_callback = ["datadog"] litellm.set_verbose = True response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "what llm are u"}], max_tokens=10, temperature=0.2, diff --git a/tests/logging_callback_tests/test_datadog_llm_obs.py b/tests/logging_callback_tests/test_datadog_llm_obs.py index 74f642e6fa..56aae7aa8b 100644 --- a/tests/logging_callback_tests/test_datadog_llm_obs.py +++ b/tests/logging_callback_tests/test_datadog_llm_obs.py @@ -48,9 +48,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", @@ -93,7 +93,7 @@ async def test_datadog_llm_obs_logging(): for _ in range(2): response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello testing dd llm obs!"}], mock_response="hi", ) diff --git a/tests/logging_callback_tests/test_generic_api_callback.py b/tests/logging_callback_tests/test_generic_api_callback.py index 6984b6fa00..fbe74d017a 100644 --- a/tests/logging_callback_tests/test_generic_api_callback.py +++ b/tests/logging_callback_tests/test_generic_api_callback.py @@ -59,7 +59,7 @@ async def test_generic_api_callback(): # Make the completion call response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="hi", user="test_user", @@ -109,11 +109,11 @@ async def test_generic_api_callback(): # Basic assertions for standard logging payload assert payload_item["response_cost"] > 0, "Response cost should be greater than 0" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" assert ( payload_item["model_parameters"]["user"] == "test_user" ), "User should be test_user" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" assert payload_item["messages"] == [ {"role": "user", "content": "Hello, world!"} ], "Messages should be the same" @@ -147,7 +147,7 @@ async def test_generic_api_callback_multiple_logs(): # Make the completion call for _ in range(10): response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="hi", user="test_user", @@ -197,11 +197,11 @@ async def test_generic_api_callback_multiple_logs(): assert ( payload_item["response_cost"] > 0 ), "Response cost should be greater than 0" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" assert ( payload_item["model_parameters"]["user"] == "test_user" ), "User should be test_user" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" assert payload_item["messages"] == [ {"role": "user", "content": "Hello, world!"} ], "Messages should be the same" @@ -239,7 +239,7 @@ async def test_generic_api_callback_ndjson_format(): # Make multiple completion calls to generate multiple logs for i in range(3): response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": f"Hello, world! {i}"}], mock_response="hi", user="test_user", @@ -279,7 +279,7 @@ async def test_generic_api_callback_ndjson_format(): assert ( payload_item["response_cost"] > 0 ), "Response cost should be greater than 0" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" assert ( payload_item["model_parameters"]["user"] == "test_user" ), "User should be test_user" @@ -314,7 +314,7 @@ async def test_generic_api_callback_single_format(): # Make 3 completion calls for i in range(3): response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": f"Hello, world! {i}"}], mock_response="hi", user="test_user", @@ -345,7 +345,7 @@ async def test_generic_api_callback_single_format(): assert ( payload_item["response_cost"] > 0 ), "Response cost should be greater than 0" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" @pytest.mark.asyncio @@ -377,7 +377,7 @@ async def test_generic_api_callback_json_array_format_explicit(): # Make multiple completion calls for i in range(5): response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": f"Hello, world! {i}"}], mock_response="hi", user="test_user", @@ -404,7 +404,7 @@ async def test_generic_api_callback_json_array_format_explicit(): assert ( payload_item["response_cost"] > 0 ), "Response cost should be greater than 0" - assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o" + assert payload_item["model"] == "gpt-5.5", "Model should be gpt-5.5" @pytest.mark.asyncio @@ -434,7 +434,7 @@ async def test_generic_api_callback_sumologic_uses_ndjson(): # Make completion calls for i in range(2): await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": f"Test {i}"}], mock_response="response", user="test_user", diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 612dbc1bfb..547e9d15f0 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -40,9 +40,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", diff --git a/tests/logging_callback_tests/test_langsmith_unit_test.py b/tests/logging_callback_tests/test_langsmith_unit_test.py index 155b1f396f..9cc1acd1ee 100644 --- a/tests/logging_callback_tests/test_langsmith_unit_test.py +++ b/tests/logging_callback_tests/test_langsmith_unit_test.py @@ -332,7 +332,7 @@ async def test_langsmith_key_based_logging(): litellm.callbacks = [LangsmithLogger()] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "Test message"}], max_tokens=10, temperature=0.2, @@ -373,7 +373,7 @@ async def test_langsmith_key_based_logging(): "inputs": { "id": "chatcmpl-82699ee4-7932-4fc0-9585-76abc8caeafa", "call_type": "acompletion", - "model": "gpt-3.5-turbo", + "model": "gpt-4.1-mini", "messages": [{"role": "user", "content": "Test message"}], "model_parameters": { "temperature": 0.2, @@ -382,7 +382,7 @@ async def test_langsmith_key_based_logging(): }, "outputs": { "id": "chatcmpl-82699ee4-7932-4fc0-9585-76abc8caeafa", - "model": "gpt-3.5-turbo", + "model": "gpt-4.1-mini", "choices": [ { "finish_reason": "stop", @@ -468,7 +468,7 @@ async def test_langsmith_queue_logging(): # Make multiple calls to ensure we don't hit the batch size for _ in range(5): response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "Test message"}], max_tokens=10, temperature=0.2, @@ -487,7 +487,7 @@ async def test_langsmith_queue_logging(): # Now make calls to exceed the batch size for _ in range(3): response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "Test message"}], max_tokens=10, temperature=0.2, diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 08d0abd272..3f4b446bea 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -39,7 +39,7 @@ async def test_global_redaction_on(): test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hello", ) @@ -69,7 +69,7 @@ async def test_global_redaction_ignores_dynamic_param(turn_off_message_logging): test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], turn_off_message_logging=turn_off_message_logging, mock_response="hello", @@ -101,7 +101,7 @@ async def test_global_redaction_off_ignores_dynamic_param(turn_off_message_loggi test_custom_logger = TestCustomLogger() litellm.callbacks = [test_custom_logger] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], turn_off_message_logging=turn_off_message_logging, mock_response="hello", @@ -129,7 +129,7 @@ async def test_redaction_responses_api(): litellm.callbacks = [test_custom_logger] response = await litellm.aresponses( - model="gpt-3.5-turbo", + model="gpt-5-mini", input="hi", mock_response="This is a test response", ) @@ -198,7 +198,7 @@ async def test_redaction_responses_api_stream(): new=mock_post, ): response = await litellm.aresponses( - model="gpt-3.5-turbo", + model="gpt-5-mini", input="hi", stream=True, ) @@ -411,7 +411,7 @@ async def test_redaction_with_streaming_response(): # This simulates the scenario where a streaming response returns a coroutine # that would normally cause the pickle error response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=True, mock_response="hello", @@ -450,7 +450,7 @@ async def test_disable_redaction_header_responses_api(): # Pass the header via litellm_metadata (as the proxy does for Responses API) response = await litellm.aresponses( - model="gpt-3.5-turbo", + model="gpt-5-mini", input="hi", mock_response="This is a test response", litellm_metadata={"headers": {"litellm-disable-message-redaction": "true"}}, @@ -487,7 +487,7 @@ async def test_redaction_with_metadata_completion_api(): # to determine which field to check. No headers means redaction should happen # based on the global setting (litellm.turn_off_message_logging = True) response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hello", metadata={}, diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index 880fac5f67..e8ca84a78a 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -53,7 +53,7 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): litellm.callbacks = ["otel"] await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, world!"}], mock_response="Hey!", metadata={"litellm_parent_otel_span": parent_otel_span}, diff --git a/tests/logging_callback_tests/test_otel_logging.py b/tests/logging_callback_tests/test_otel_logging.py index ea1c884c32..b6d7ef4be4 100644 --- a/tests/logging_callback_tests/test_otel_logging.py +++ b/tests/logging_callback_tests/test_otel_logging.py @@ -48,7 +48,7 @@ async def test_async_otel_callback(streaming): litellm.callbacks = [OpenTelemetry(config=OpenTelemetryConfig(exporter=exporter))] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], temperature=0.1, user="OTEL_USER", @@ -76,7 +76,7 @@ async def test_async_otel_callback(streaming): if span.name == "litellm_request": validate_litellm_request(span) # Additional specific checks - assert span._attributes["gen_ai.request.model"] == "gpt-3.5-turbo" + assert span._attributes["gen_ai.request.model"] == "gpt-4.1-mini" assert span._attributes["gen_ai.system"] == "openai" assert span._attributes["gen_ai.request.temperature"] == 0.1 assert span._attributes["llm.is_streaming"] == str(streaming) @@ -185,7 +185,7 @@ async def test_awesome_otel_with_message_logging_off(streaming, global_redact): litellm.failure_callback = [] response = await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], mock_response="hi", stream=streaming, @@ -293,7 +293,7 @@ async def test_arize_phoenix_creates_nested_spans_on_dedicated_provider(): # Simulate a proxy request by injecting proxy_server_request as a top-level kwarg. # This triggers ArizePhoenixLogger._get_phoenix_context to create its own parent span. await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-4.1-mini", messages=[{"role": "user", "content": "ping"}], mock_response="pong", proxy_server_request={ diff --git a/tests/logging_callback_tests/test_pagerduty_alerting.py b/tests/logging_callback_tests/test_pagerduty_alerting.py index 33c24102eb..108a1ead1a 100644 --- a/tests/logging_callback_tests/test_pagerduty_alerting.py +++ b/tests/logging_callback_tests/test_pagerduty_alerting.py @@ -27,7 +27,7 @@ async def test_pagerduty_alerting(): try: await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="litellm.RateLimitError", ) @@ -48,7 +48,7 @@ async def test_pagerduty_alerting_high_failure_rate(): try: await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="litellm.RateLimitError", ) @@ -61,7 +61,7 @@ async def test_pagerduty_alerting_high_failure_rate(): for _ in range(3): try: await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="litellm.RateLimitError", ) @@ -88,12 +88,12 @@ async def test_pagerduty_hanging_request_alerting(): user_id="test-user", end_user_id="test-end-user", ), - data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + data={"model": "gpt-5.5", "messages": [{"role": "user", "content": "hi"}]}, call_type="completion", ) await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], ) diff --git a/tests/logging_callback_tests/test_posthog.py b/tests/logging_callback_tests/test_posthog.py index 344b8c7166..b3f346bcf9 100644 --- a/tests/logging_callback_tests/test_posthog.py +++ b/tests/logging_callback_tests/test_posthog.py @@ -33,7 +33,7 @@ def create_standard_logging_payload() -> StandardLoggingPayload: "endTime": 1234567891.0, "completionStartTime": 1234567890.5, "response_time": 1.0, - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "model_id": "model-123", "api_base": "https://api.openai.com", "cache_hit": False, @@ -57,7 +57,7 @@ async def test_create_posthog_event_payload(): event_payload = posthog_logger.create_posthog_event_payload(kwargs) assert event_payload["event"] == "$ai_generation" - assert event_payload["properties"]["$ai_model"] == "gpt-3.5-turbo" + assert event_payload["properties"]["$ai_model"] == "gpt-5-mini" assert event_payload["properties"]["$ai_input_tokens"] == 20 assert event_payload["properties"]["$ai_output_tokens"] == 10 @@ -251,7 +251,7 @@ async def test_custom_metadata_with_no_metadata(): # Should not error and should have standard properties assert event_payload["event"] == "$ai_generation" - assert event_payload["properties"]["$ai_model"] == "gpt-3.5-turbo" + assert event_payload["properties"]["$ai_model"] == "gpt-5-mini" # Test with empty metadata kwargs = { @@ -262,7 +262,7 @@ async def test_custom_metadata_with_no_metadata(): # Should not error and should have standard properties assert event_payload["event"] == "$ai_generation" - assert event_payload["properties"]["$ai_model"] == "gpt-3.5-turbo" + assert event_payload["properties"]["$ai_model"] == "gpt-5-mini" @pytest.mark.asyncio diff --git a/tests/logging_callback_tests/test_spend_logs.py b/tests/logging_callback_tests/test_spend_logs.py index 131de5992f..f9c4db7c6d 100644 --- a/tests/logging_callback_tests/test_spend_logs.py +++ b/tests/logging_callback_tests/test_spend_logs.py @@ -91,7 +91,7 @@ def test_spend_logs_payload(model_id: Optional[str]): "content-length": "163", }, "endpoint": "http://localhost:4000/chat/completions", - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", "deployment": "azure/gpt-4.1-mini", "model_info": { "id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4", @@ -129,7 +129,7 @@ def test_spend_logs_payload(model_id: Optional[str]): }, {"role": "user", "content": "bom dia"}, ], - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "max_tokens": 10, }, }, @@ -332,7 +332,7 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch): input_args: dict = { "kwargs": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello!"}], "litellm_params": { "metadata": { @@ -349,7 +349,7 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch): message=litellm.Message(content="Hi there!", role="assistant"), ) ], - model="gpt-3.5-turbo", + model="gpt-5-mini", usage=litellm.Usage(completion_tokens=2, prompt_tokens=1, total_tokens=3), ), "start_time": datetime.datetime.now(), @@ -372,7 +372,7 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch): litellm_params = { "proxy_server_request": { "body": { - "model": "gpt-4", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Hello!"}], } } @@ -389,7 +389,7 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch): {"role": "assistant", "content": "Hi there!"} ) proxy_server_request = json.loads(payload["proxy_server_request"] or "{}") - assert proxy_server_request["model"] == "gpt-4" + assert proxy_server_request["model"] == "gpt-5.5" assert proxy_server_request["messages"] == [{"role": "user", "content": "Hello!"}] # Clean up - reset general_settings @@ -420,7 +420,7 @@ def test_large_request_no_truncation_threshold(): request_body = { "messages": [{"role": "user", "content": large_content}], - "model": "gpt-4", + "model": "gpt-5.5", } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) @@ -454,7 +454,7 @@ def test_small_request_no_truncation(): request_body = { "messages": [{"role": "user", "content": small_content}], - "model": "gpt-4", + "model": "gpt-5.5", } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) @@ -497,7 +497,7 @@ def test_configurable_string_length_env_var(monkeypatch): request_body = { "messages": [{"role": "user", "content": large_content}], - "model": "gpt-4", + "model": "gpt-5.5", } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) @@ -531,7 +531,7 @@ def test_truncation_preserves_beginning_and_end(): request_body = { "messages": [{"role": "user", "content": large_content}], - "model": "gpt-4", + "model": "gpt-5.5", } sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) diff --git a/tests/logging_callback_tests/test_sqs_logger.py b/tests/logging_callback_tests/test_sqs_logger.py index 3403a7b595..83692af3bc 100644 --- a/tests/logging_callback_tests/test_sqs_logger.py +++ b/tests/logging_callback_tests/test_sqs_logger.py @@ -34,7 +34,7 @@ async def test_async_sqs_logger_flush(): litellm.callbacks = [sqs_logger] await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "hello"}], mock_response="hi", ) @@ -74,7 +74,7 @@ async def test_async_sqs_logger_flush(): assert "model" in payload_data assert "messages" in payload_data assert "response" in payload_data - assert payload_data["model"] == "gpt-4o" + assert payload_data["model"] == "gpt-5.5" assert len(payload_data["messages"]) == 1 assert payload_data["messages"][0]["role"] == "user" assert payload_data["messages"][0]["content"] == "hello" @@ -99,7 +99,7 @@ async def test_async_sqs_logger_error_flush(): litellm.callbacks = [sqs_logger] await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "hello"}], mock_response="Error occurred", ) @@ -139,7 +139,7 @@ async def test_async_sqs_logger_error_flush(): assert "model" in payload_data assert "messages" in payload_data assert "response" in payload_data - assert payload_data["model"] == "gpt-4o" + assert payload_data["model"] == "gpt-5.5" assert len(payload_data["messages"]) == 1 assert payload_data["messages"][0]["role"] == "user" assert payload_data["messages"][0]["content"] == "hello" diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index ea1f84b11e..36215ca9c6 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -317,16 +317,16 @@ def test_get_model_cost_information(): # Test with valid model result = StandardLoggingPayloadSetup.get_model_cost_information( - base_model="gpt-3.5-turbo", + base_model="gpt-5-mini", custom_pricing=False, custom_llm_provider="openai", init_response_obj={}, ) litellm_info_gpt_3_5_turbo_model_map_value = litellm.get_model_info( - model="gpt-3.5-turbo", custom_llm_provider="openai" + model="gpt-5-mini", custom_llm_provider="openai" ) print("result", result) - assert result["model_map_key"] == "gpt-3.5-turbo" + assert result["model_map_key"] == "gpt-5-mini" assert result["model_map_value"] is not None assert result["model_map_value"] == litellm_info_gpt_3_5_turbo_model_map_value # assert all fields in StandardLoggingModelInformation are present @@ -515,7 +515,7 @@ def test_get_error_information(): litellm_exception = litellm.exceptions.RateLimitError( message="Test error", llm_provider="openai", - model="gpt-3.5-turbo", + model="gpt-5-mini", response=None, litellm_debug_info=None, max_retries=None, @@ -603,7 +603,7 @@ def test_cost_breakdown_in_standard_logging_payload(): # Create a mock logging object with cost breakdown logging_obj = Logging( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello"}], stream=False, call_type="completion", @@ -624,7 +624,7 @@ def test_cost_breakdown_in_standard_logging_payload(): mock_response = { "id": "chatcmpl-123", "object": "chat.completion", - "model": "gpt-4o", + "model": "gpt-5.5", "usage": { "prompt_tokens": 10, "completion_tokens": 20, @@ -644,7 +644,7 @@ def test_cost_breakdown_in_standard_logging_payload(): # Create kwargs kwargs = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Hello"}], "response_cost": 0.0035, "custom_llm_provider": "openai", @@ -687,7 +687,7 @@ def test_cost_breakdown_missing_in_standard_logging_payload(): # Create a mock logging object without cost breakdown logging_obj = Logging( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello"}], stream=False, call_type="embedding", # Non-completion call type @@ -702,12 +702,12 @@ def test_cost_breakdown_missing_in_standard_logging_payload(): mock_response = { "object": "list", "data": [{"embedding": [0.1, 0.2, 0.3]}], - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "usage": {"prompt_tokens": 10, "total_tokens": 10}, } kwargs = { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "input": ["Hello"], "response_cost": 0.0001, "custom_llm_provider": "openai", @@ -756,7 +756,7 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): from datetime import datetime logging_obj = Logging( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hi"}], stream=False, call_type="completion", @@ -768,7 +768,7 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): mock_response = { "id": "chatcmpl-usage-test", "object": "chat.completion", - "model": "gpt-4o", + "model": "gpt-5.5", "usage": { "prompt_tokens": 42, "completion_tokens": 58, @@ -784,7 +784,7 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): } kwargs = { - "model": "gpt-4o", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "Hi"}], "response_cost": 0.01, "custom_llm_provider": "openai", diff --git a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py index a077c76f61..4088bdd2cf 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py +++ b/tests/logging_callback_tests/test_standard_logging_payload_excluded_fields.py @@ -49,7 +49,7 @@ def create_sample_standard_logging_payload() -> Dict: "completionStartTime": 1234567890.5, "response_time": 1.0, "model_map_information": {}, - "model": "gpt-4", + "model": "gpt-5.5", "model_id": "model-123", "model_group": None, "api_base": "https://api.openai.com/v1", diff --git a/tests/logging_callback_tests/test_token_counting.py b/tests/logging_callback_tests/test_token_counting.py index 4c8efa4989..69200f113d 100644 --- a/tests/logging_callback_tests/test_token_counting.py +++ b/tests/logging_callback_tests/test_token_counting.py @@ -55,7 +55,7 @@ async def test_stream_token_counting_gpt_4o(): litellm.logging_callback_manager.add_litellm_callback(custom_logger) response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, how are you?" * 100}], stream=True, stream_options={"include_usage": True}, @@ -95,7 +95,7 @@ async def test_stream_token_counting_without_include_usage(): litellm.logging_callback_manager.add_litellm_callback(custom_logger) response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, how are you?" * 100}], stream=True, ) @@ -133,7 +133,7 @@ async def test_stream_token_counting_with_redaction(): litellm.logging_callback_manager.add_litellm_callback(custom_logger) response = await litellm.acompletion( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, how are you?" * 100}], stream=True, ) diff --git a/tests/logging_callback_tests/test_unit_test_litellm_logging.py b/tests/logging_callback_tests/test_unit_test_litellm_logging.py index 455d0dacb9..e01c09951d 100644 --- a/tests/logging_callback_tests/test_unit_test_litellm_logging.py +++ b/tests/logging_callback_tests/test_unit_test_litellm_logging.py @@ -27,7 +27,7 @@ service_logger = ServiceLogging() def setup_logging(): return Logging( - model="gpt-4o", + model="gpt-5.5", messages=[{"role": "user", "content": "Hello, world!"}], stream=False, call_type="completion", diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index 6f6efdd202..b2243eed04 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -164,7 +164,7 @@ async def use_callback_in_llm_call( for _ in range(5): await litellm.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], temperature=0.1, mock_response="hello", @@ -217,7 +217,7 @@ def test_dynamic_logging_global_callback(): cl = CustomLogger() litellm_logging = LiteLLMLoggingObj( - model="claude-3-opus-20240229", + model="claude-opus-4-7", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="completion", @@ -240,7 +240,7 @@ def test_dynamic_logging_global_callback(): result=ModelResponse( id="chatcmpl-5418737b-ab14-420b-b9c5-b278b6681b70", created=1732306261, - model="claude-3-opus-20240229", + model="claude-opus-4-7", object="chat.completion", system_fingerprint=None, choices=[ @@ -277,7 +277,7 @@ def test_get_combined_callback_list(): from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _logging = LiteLLMLoggingObj( - model="claude-3-opus-20240229", + model="claude-opus-4-7", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="completion", @@ -298,7 +298,7 @@ def test_get_combined_callback_list_returns_copy_when_dynamic_is_none(): from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _logging = LiteLLMLoggingObj( - model="claude-3-opus-20240229", + model="claude-opus-4-7", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="completion", diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py index 94a3f5d331..220a44f079 100644 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py @@ -74,7 +74,7 @@ def validate_stream_chunk(chunk): def test_basic_response(): client = get_test_client() response = client.responses.create( - model="gpt-4o", input="just respond with the word 'ping'" + model="gpt-5.5", input="just respond with the word 'ping'" ) print("basic response=", response) @@ -94,7 +94,7 @@ def test_basic_response(): def test_streaming_response(): client = get_test_client() stream = client.responses.create( - model="gpt-4o", input="just respond with the word 'ping'", stream=True + model="gpt-5.5", input="just respond with the word 'ping'", stream=True ) collected_chunks = [] @@ -117,7 +117,7 @@ def test_bad_request_bad_param_error(): with pytest.raises(BadRequestError): # Trigger error with invalid model name client.responses.create( - model="gpt-4o", input="This should fail", temperature=2000 + model="gpt-5.5", input="This should fail", temperature=2000 ) @@ -137,7 +137,7 @@ def test_cancel_response(): from litellm.types.llms.openai import ResponsesAPIResponse response = client.responses.create( - model="gpt-4o", input="just respond with the word 'ping'", background=True + model="gpt-5.5", input="just respond with the word 'ping'", background=True ) print("basic response=", response) @@ -160,7 +160,7 @@ def test_cancel_streaming_response(): from litellm.types.llms.openai import ResponsesAPIResponse stream = client.responses.create( - model="gpt-4o", + model="gpt-5.5", input="just respond with the word 'ping'", stream=True, background=True, diff --git a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py index ad28e8da3d..c6f4128f2c 100644 --- a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py +++ b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py @@ -233,8 +233,8 @@ async def test_list_batches_with_target_model_names(): """ # Test data - target_model_names = "gpt-4,gpt-3.5-turbo" - expected_model = "gpt-4" # Should use the first model from the comma-separated list + target_model_names = "gpt-5.5,gpt-5-mini" + expected_model = "gpt-5.5" # Should use the first model from the comma-separated list # Mock response for list_batches mock_batch_response = { diff --git a/tests/openai_endpoints_tests/test_openai_fine_tuning.py b/tests/openai_endpoints_tests/test_openai_fine_tuning.py index 108e336df3..8d46692a80 100644 --- a/tests/openai_endpoints_tests/test_openai_fine_tuning.py +++ b/tests/openai_endpoints_tests/test_openai_fine_tuning.py @@ -30,7 +30,7 @@ async def test_openai_fine_tuning(): # create fine tuning job ft_job = await client.fine_tuning.jobs.create( - model="gpt-4o-mini-2024-07-18", + model="gpt-4.1-mini-2025-04-14", training_file=response.id, extra_headers={"custom-llm-provider": "openai"}, ) diff --git a/tests/openai_endpoints_tests/test_responses_websocket_proxy_e2e.py b/tests/openai_endpoints_tests/test_responses_websocket_proxy_e2e.py index e8d1814de7..ab05442d00 100644 --- a/tests/openai_endpoints_tests/test_responses_websocket_proxy_e2e.py +++ b/tests/openai_endpoints_tests/test_responses_websocket_proxy_e2e.py @@ -6,7 +6,7 @@ and validates the streamed response events. Requires: - Proxy running: python -m litellm.proxy.proxy_cli --config --port 4000 - - Model configured in proxy (e.g. gpt-4o-mini) + - Model configured in proxy (e.g. gpt-5-mini) See: https://developers.openai.com/api/docs/guides/websocket-mode/ """ @@ -21,7 +21,7 @@ import pytest # ── Configuration ───────────────────────────────────────────────────────────── PROXY_BASE_URL = os.environ.get("LITELLM_PROXY_BASE_URL", "ws://0.0.0.0:4000") PROXY_MASTER_KEY = os.environ.get("LITELLM_PROXY_KEY", "sk-1234") -PROXY_MODEL = os.environ.get("LITELLM_PROXY_RESPONSES_MODEL", "gpt-4o-mini") +PROXY_MODEL = os.environ.get("LITELLM_PROXY_RESPONSES_MODEL", "gpt-5-mini") # ────────────────────────────────────────────────────────────────────────────── diff --git a/tests/otel_tests/test_e2e_model_access.py b/tests/otel_tests/test_e2e_model_access.py index 87d85a1960..5b5f2a89c8 100644 --- a/tests/otel_tests/test_e2e_model_access.py +++ b/tests/otel_tests/test_e2e_model_access.py @@ -59,12 +59,12 @@ async def mock_chat_completion(session, key: str, model: str): "key_models, test_model, expect_success", [ (["openai/*"], "anthropic/claude-2", False), # Non-matching model - (["gpt-4"], "gpt-4", True), # Exact model match + (["gpt-5.5"], "gpt-5.5", True), # Exact model match (["bedrock/*"], "bedrock/anthropic.claude-3", True), # Bedrock wildcard (["bedrock/anthropic.*"], "bedrock/anthropic.claude-3", True), # Pattern match (["bedrock/anthropic.*"], "bedrock/amazon.titan", False), # Pattern non-match - (None, "gpt-4", True), # No model restrictions - ([], "gpt-4", True), # Empty model list + (None, "gpt-5.5", True), # No model restrictions + ([], "gpt-5.5", True), # Empty model list ], ) @pytest.mark.asyncio @@ -119,7 +119,7 @@ async def test_model_access_update(): response = await client.post( "/key/generate", json={ - "models": ["openai/gpt-4"], + "models": ["openai/gpt-5.5"], "metadata": dict(_ALLOW_CLIENT_MOCK_METADATA), }, headers=headers, @@ -130,13 +130,13 @@ async def test_model_access_update(): # Test initial access async with aiohttp.ClientSession() as session: - # Should work with gpt-4 - await mock_chat_completion(session=session, key=key, model="openai/gpt-4") + # Should work with gpt-5.5 + await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") - # Should fail with gpt-3.5-turbo + # Should fail with gpt-5-mini with pytest.raises(Exception) as exc_info: await mock_chat_completion( - session=session, key=key, model="openai/gpt-3.5-turbo" + session=session, key=key, model="openai/gpt-5-mini" ) _validate_model_access_exception( exc_info.value, expected_type="key_model_access_denied" @@ -151,9 +151,9 @@ async def test_model_access_update(): # Test updated access async with aiohttp.ClientSession() as session: # Both models should now work - await mock_chat_completion(session=session, key=key, model="openai/gpt-4") + await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") await mock_chat_completion( - session=session, key=key, model="openai/gpt-3.5-turbo" + session=session, key=key, model="openai/gpt-5-mini" ) # Non-OpenAI model should still fail @@ -226,7 +226,7 @@ async def test_team_model_access_update(): response = await client.post( "/team/new", json={ - "models": ["openai/gpt-4"], + "models": ["openai/gpt-5.5"], "name": "test-team", "metadata": dict(_ALLOW_CLIENT_MOCK_METADATA), }, @@ -250,13 +250,13 @@ async def test_team_model_access_update(): # Test initial access async with aiohttp.ClientSession() as session: - # Should work with gpt-4 - await mock_chat_completion(session=session, key=key, model="openai/gpt-4") + # Should work with gpt-5.5 + await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") - # Should fail with gpt-3.5-turbo + # Should fail with gpt-5-mini with pytest.raises(Exception) as exc_info: await mock_chat_completion( - session=session, key=key, model="openai/gpt-3.5-turbo" + session=session, key=key, model="openai/gpt-5-mini" ) _validate_model_access_exception( exc_info.value, expected_type="team_model_access_denied" @@ -273,9 +273,9 @@ async def test_team_model_access_update(): # Test updated access async with aiohttp.ClientSession() as session: # Both models should now work - await mock_chat_completion(session=session, key=key, model="openai/gpt-4") + await mock_chat_completion(session=session, key=key, model="openai/gpt-5.5") await mock_chat_completion( - session=session, key=key, model="openai/gpt-3.5-turbo" + session=session, key=key, model="openai/gpt-5-mini" ) # Non-OpenAI model should still fail diff --git a/tests/otel_tests/test_guardrails.py b/tests/otel_tests/test_guardrails.py index 08c82d1630..ecc5d2eda5 100644 --- a/tests/otel_tests/test_guardrails.py +++ b/tests/otel_tests/test_guardrails.py @@ -11,7 +11,7 @@ async def chat_completion( session, key, messages, - model: Union[str, List] = "gpt-4", + model: Union[str, List] = "gpt-5.5", guardrails: Optional[List] = None, ): url = "http://0.0.0.0:4000/chat/completions" diff --git a/tests/otel_tests/test_otel.py b/tests/otel_tests/test_otel.py index 9ded859eb9..af191b46b6 100644 --- a/tests/otel_tests/test_otel.py +++ b/tests/otel_tests/test_otel.py @@ -11,8 +11,8 @@ from litellm._uuid import uuid async def generate_key( session, models=[ - "gpt-4", - "text-embedding-ada-002", + "gpt-5.5", + "text-embedding-3-small", "gpt-image-1", "fake-openai-endpoint", "mistral-embed", @@ -38,7 +38,7 @@ async def generate_key( return await response.json() -async def chat_completion(session, key, model: Union[str, List] = "gpt-4"): +async def chat_completion(session, key, model: Union[str, List] = "gpt-5.5"): url = "http://0.0.0.0:4000/chat/completions" headers = { "Authorization": f"Bearer {key}", diff --git a/tests/otel_tests/test_prometheus.py b/tests/otel_tests/test_prometheus.py index 75061dda94..c9490af07c 100644 --- a/tests/otel_tests/test_prometheus.py +++ b/tests/otel_tests/test_prometheus.py @@ -177,7 +177,7 @@ async def test_proxy_failure_metrics(): @pytest.mark.flaky(retries=3, delay=2) async def test_proxy_success_metrics(): """ - Make 1 good /chat/completions call to "openai/gpt-3.5-turbo" + Make 1 good /chat/completions call to "openai/gpt-5-mini" GET /metrics Assert the success metric is incremented by 1 """ diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py index 8b52cedf37..64acc68c26 100644 --- a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py @@ -98,9 +98,9 @@ class BaseAnthropicMessagesToolSearchTest(ABC): Returns the model string to use for tests. Examples: - - "anthropic/claude-sonnet-4-20250514" - - "vertex_ai/claude-sonnet-4@20250514" - - "bedrock/invoke/anthropic.claude-sonnet-4-20250514-v1:0" + - "anthropic/claude-sonnet-4-5-20250929" + - "vertex_ai/claude-sonnet-4-5@20250929" + - "bedrock/invoke/anthropic.claude-sonnet-4-5-20250929-v1:0" """ pass diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 84b14f9508..8ea9506095 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -112,7 +112,7 @@ class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): @property def model_config(self) -> Dict[str, Any]: return { - "model": "openai/gpt-4o-mini", + "model": "openai/gpt-4.1-mini", "client": None, } @@ -121,7 +121,7 @@ class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): """ This is the model name that is expected to be in the logging payload """ - return "gpt-4o-mini" + return "gpt-4.1-mini" @pytest.mark.asyncio async def test_anthropic_messages_litellm_router_streaming_with_logging(self): @@ -283,23 +283,23 @@ async def test_anthropic_messages_fallbacks(): router = Router( model_list=[ { - "model_name": "anthropic/claude-opus-4-20250514", + "model_name": "anthropic/claude-opus-4-7", "litellm_params": { - "model": "anthropic/claude-opus-4-20250514", + "model": "anthropic/claude-opus-4-7", "api_key": "bad-key", }, }, { - "model_name": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model_name": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "litellm_params": { - "model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", }, }, ], fallbacks=[ { - "anthropic/claude-opus-4-20250514": [ - "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + "anthropic/claude-opus-4-7": [ + "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" ] } ], @@ -311,7 +311,7 @@ async def test_anthropic_messages_fallbacks(): # Call the handler response = await router.aanthropic_messages( messages=messages, - model="anthropic/claude-opus-4-20250514", + model="anthropic/claude-opus-4-7", max_tokens=100, metadata={ "user_id": "hello", @@ -871,7 +871,7 @@ def test_sync_openai_messages(): litellm._turn_on_debug() response = litellm.anthropic.messages.create( messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], - model="openai/gpt-4o-mini", + model="openai/gpt-4.1-mini", max_tokens=100, ) print("ANT response", response) diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py b/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py index 8d6c05adef..c8b91c3c49 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_tool_search.py @@ -50,7 +50,7 @@ class TestAnthropicAPIToolSearch(BaseAnthropicMessagesToolSearchTest): # """ # def get_model(self) -> str: -# return "azure/claude-sonnet-4-20250514" +# return "azure/claude-sonnet-4-5-20250929" # class TestVertexAIToolSearch(BaseAnthropicMessagesToolSearchTest): diff --git a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py index e629156142..dcc44cae77 100644 --- a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py +++ b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py @@ -28,15 +28,15 @@ async def test_anthropic_messages_litellm_router_bedrock(): router = Router( model_list=[ { - "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "litellm_params": { - "model": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", }, }, { - "model_name": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model_name": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "litellm_params": { - "model": "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", }, }, ] @@ -45,20 +45,20 @@ async def test_anthropic_messages_litellm_router_bedrock(): # Set up test parameters messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - # Call 1 using bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0 + # Call 1 using bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0 response = await router.aanthropic_messages( messages=messages, - model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", max_tokens=100, ) # Verify response INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) - # Call 2 using bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0 + # Call 2 using bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 response = await router.aanthropic_messages( messages=messages, - model="bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0", + model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", max_tokens=100, ) @@ -75,9 +75,9 @@ async def test_anthropic_messages_bedrock_converse_with_thinking(): router = Router( model_list=[ { - "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "litellm_params": { - "model": "bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + "model": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", }, }, ] @@ -87,7 +87,7 @@ async def test_anthropic_messages_bedrock_converse_with_thinking(): response = await router.aanthropic_messages( messages=messages, - model="bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0", + model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", max_tokens=1026, thinking={"type": "enabled", "budget_tokens": 1025}, ) diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index 14b3d9b71b..6e6507f982 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -45,7 +45,7 @@ async def test_assistants_passthrough_logging(): "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", "name": "Math Tutor", "tools": [{"type": "code_interpreter"}], - "model": "gpt-4o", + "model": "gpt-4.1-mini", } TARGET_METHOD = "POST" diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index cfdd8a4e3c..1b16177b75 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -451,7 +451,7 @@ def test_init_kwargs_filters_pricing_params(mock_request, mock_user_api_key_dict # Create a parsed body with pricing parameters that should be filtered out parsed_body = { - "model": "gpt-4", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "test"}], # Standard pricing params (should be filtered) "input_cost_per_token": 0.00002, @@ -491,7 +491,7 @@ def test_init_kwargs_filters_pricing_params(mock_request, mock_user_api_key_dict _parsed_body=parsed_body, litellm_call_id="test-call-id", logging_obj=LiteLLMLoggingObj( - model="gpt-4", + model="gpt-5.5", messages=[{"role": "user", "content": "test"}], stream=False, call_type="completion", @@ -520,7 +520,7 @@ def test_init_kwargs_filters_pricing_params(mock_request, mock_user_api_key_dict assert "tiered_pricing" not in parsed_body # Verify valid OpenAI parameters remain in parsed_body - assert parsed_body["model"] == "gpt-4" + assert parsed_body["model"] == "gpt-5.5" assert parsed_body["messages"] == [{"role": "user", "content": "test"}] assert parsed_body["temperature"] == 0.7 assert parsed_body["max_tokens"] == 100 @@ -560,7 +560,7 @@ def test_custom_pricing_used_in_cost_calculation(): ) ], created=1234567890, - model="gpt-4", + model="gpt-5.5", object="chat.completion", usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), ) @@ -568,7 +568,7 @@ def test_custom_pricing_used_in_cost_calculation(): # Test 1: Standard pricing (should use default model pricing) standard_cost = completion_cost( completion_response=resp, - model="gpt-4", + model="gpt-5.5", ) print(f"Standard cost: {standard_cost}") diff --git a/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py b/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py index 97a1f2eecc..455c72ff63 100644 --- a/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py +++ b/tests/pass_through_unit_tests/test_unit_test_anthropic_pass_through.py @@ -23,7 +23,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passth @pytest.fixture def mock_response(): return { - "model": "claude-3-opus-20240229", + "model": "claude-opus-4-7", "content": [{"text": "Hello, world!", "type": "text"}], "role": "assistant", } @@ -50,7 +50,7 @@ def mock_httpx_response(): @pytest.fixture def mock_logging_obj(): logging_obj = LiteLLMLoggingObj( - model="claude-3-opus-20240229", + model="claude-opus-4-7", messages=[], stream=False, call_type="completion", @@ -101,7 +101,7 @@ def test_create_anthropic_response_logging_payload(mock_logging_obj, metadata_pa result = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( litellm_model_response=model_response, - model="claude-3-opus-20240229", + model="claude-opus-4-7", kwargs={ "litellm_params": { "metadata": { @@ -249,7 +249,7 @@ def test_get_user_from_metadata(end_user_id): def all_chunks(): return [ "event: message_start", - 'data: {"type":"message_start","message":{"id":"msg_01G7T4YSBzHjmgTyizv1UfkB","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}}}', + 'data: {"type":"message_start","message":{"id":"msg_01G7T4YSBzHjmgTyizv1UfkB","type":"message","role":"assistant","model":"claude-sonnet-4-5-20250929","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":17,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}}}', "event: content_block_start", 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', "event: ping", @@ -325,7 +325,7 @@ def test_handle_logging_anthropic_collected_chunks(all_chunks): "passthrough_success_handler_obj": pass_through_logging_obj, "url_route": "https://api.anthropic.com/v1/messages", "request_body": { - "model": "claude-3-5-sonnet-20240620", + "model": "claude-sonnet-4-5-20250929", "messages": [ { "role": "user", @@ -366,7 +366,7 @@ def test_build_complete_streaming_response(all_chunks): result = AnthropicPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, - model="claude-3-5-sonnet-20240620", + model="claude-sonnet-4-5-20250929", litellm_logging_obj=litellm_logging_obj, ) diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml index e137b7ca9d..715e27e38d 100644 --- a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml +++ b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml @@ -9,9 +9,9 @@ model_list: model: "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" aws_region_name: "us-east-1" - - model_name: bedrock-claude-sonnet-4 + - model_name: bedrock-claude-sonnet-4.6 litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + model: "bedrock/us.anthropic.claude-sonnet-4-6" aws_region_name: "us-east-1" - model_name: bedrock-claude-sonnet-4.5 diff --git a/tests/router_unit_tests/create_mock_standard_logging_payload.py b/tests/router_unit_tests/create_mock_standard_logging_payload.py index 2fd6a4ffa8..106328e95e 100644 --- a/tests/router_unit_tests/create_mock_standard_logging_payload.py +++ b/tests/router_unit_tests/create_mock_standard_logging_payload.py @@ -43,9 +43,9 @@ def create_standard_logging_payload() -> StandardLoggingPayload: endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", @@ -94,9 +94,9 @@ def create_standard_logging_payload_with_long_content() -> StandardLoggingPayloa endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( - model_map_key="gpt-3.5-turbo", model_map_value=None + model_map_key="gpt-5-mini", model_map_value=None ), - model="gpt-3.5-turbo", + model="gpt-5-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", diff --git a/tests/router_unit_tests/test_completion_no_copy.py b/tests/router_unit_tests/test_completion_no_copy.py index 50e5e3b228..28f4077949 100644 --- a/tests/router_unit_tests/test_completion_no_copy.py +++ b/tests/router_unit_tests/test_completion_no_copy.py @@ -28,7 +28,7 @@ async def test_acompletion_deployment_not_mutated(): { "model_name": "gpt-3.5", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", "temperature": 0.7, }, @@ -46,7 +46,7 @@ async def test_acompletion_deployment_not_mutated(): mock_acompletion.return_value = ModelResponse( id="test", choices=[{"message": {"role": "assistant", "content": "test"}, "index": 0}], - model="gpt-3.5-turbo", + model="gpt-5-mini", usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) @@ -76,7 +76,7 @@ def test_completion_deployment_not_mutated(): { "model_name": "gpt-3.5", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", "max_tokens": 100, }, @@ -94,7 +94,7 @@ def test_completion_deployment_not_mutated(): mock_completion.return_value = ModelResponse( id="test", choices=[{"message": {"role": "assistant", "content": "test"}, "index": 0}], - model="gpt-3.5-turbo", + model="gpt-5-mini", usage={"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, ) diff --git a/tests/router_unit_tests/test_default_deployment_copy.py b/tests/router_unit_tests/test_default_deployment_copy.py index 0877ff08a3..9040147930 100644 --- a/tests/router_unit_tests/test_default_deployment_copy.py +++ b/tests/router_unit_tests/test_default_deployment_copy.py @@ -42,7 +42,7 @@ def test_default_deployment_isolation(): router.default_deployment = { # type: ignore "model_name": "default-model", "litellm_params": { - "model": "gpt-3.5-turbo", # This will be overwritten per request + "model": "gpt-5-mini", # This will be overwritten per request "api_key": "test-key", # This should be shared "custom_config": { # Deep nested - will be SHARED "nested_setting": "original", @@ -66,7 +66,7 @@ def test_default_deployment_isolation(): assert deployment2["litellm_params"]["model"] == "custom-model-2" # type: ignore # Assert: Original default_deployment must remain unchanged (not mutated by requests) - assert router.default_deployment["litellm_params"]["model"] == "gpt-3.5-turbo" # type: ignore + assert router.default_deployment["litellm_params"]["model"] == "gpt-5-mini" # type: ignore # Assert: Shared fields should still be accessible in all copies assert deployment1["litellm_params"]["api_key"] == "test-key" # type: ignore diff --git a/tests/router_unit_tests/test_get_model_list_alias_optimization.py b/tests/router_unit_tests/test_get_model_list_alias_optimization.py index 2c2df3be94..145c7e8092 100644 --- a/tests/router_unit_tests/test_get_model_list_alias_optimization.py +++ b/tests/router_unit_tests/test_get_model_list_alias_optimization.py @@ -10,18 +10,18 @@ def test_get_model_list_from_model_alias_should_not_iterate_for_non_alias_lookup router = Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, } ], - model_group_alias={"alias-1": "gpt-4"}, + model_group_alias={"alias-1": "gpt-5.5"}, ) router.model_group_alias = NoItemsAliasDict( - {f"alias-{idx}": "gpt-4" for idx in range(200)} + {f"alias-{idx}": "gpt-5.5" for idx in range(200)} ) model_alias_list = router.get_model_list_from_model_alias( - model_name="gpt-3.5-turbo" + model_name="gpt-5-mini" ) assert model_alias_list == [] @@ -30,18 +30,18 @@ def test_map_team_model_should_not_iterate_aliases_for_non_alias_team_model_name router = Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_info": { "team_id": "team-1", "team_public_model_name": "team-model", }, } ], - model_group_alias={"alias-1": "gpt-4"}, + model_group_alias={"alias-1": "gpt-5.5"}, ) router.model_group_alias = NoItemsAliasDict( - {f"alias-{idx}": "gpt-4" for idx in range(200)} + {f"alias-{idx}": "gpt-5.5" for idx in range(200)} ) # map_team_model should return the public name unchanged (not the internal UUID name) diff --git a/tests/router_unit_tests/test_pre_call_checks_optimization.py b/tests/router_unit_tests/test_pre_call_checks_optimization.py index f3d2563cbb..54d11d482a 100644 --- a/tests/router_unit_tests/test_pre_call_checks_optimization.py +++ b/tests/router_unit_tests/test_pre_call_checks_optimization.py @@ -37,13 +37,13 @@ class TestPreCallChecksOptimization: router = Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, "model_info": {"id": "test-1"}, }, { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-4", "api_key": "sk-test2"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test2"}, "model_info": {"id": "test-2"}, }, ], @@ -51,7 +51,7 @@ class TestPreCallChecksOptimization: enable_pre_call_checks=True, ) - deployments = router.get_model_list(model_name="gpt-3.5-turbo") + deployments = router.get_model_list(model_name="gpt-5-mini") assert deployments is not None # Capture the original state @@ -62,7 +62,7 @@ class TestPreCallChecksOptimization: # Call the function under test router._pre_call_checks( - model="gpt-3.5-turbo", + model="gpt-5-mini", healthy_deployments=deployments, messages=[{"role": "user", "content": "test"}], ) @@ -92,12 +92,12 @@ class TestPreCallChecksOptimization: model_list=[ { "model_name": "test", - "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-test"}, "model_info": {"id": "small", "max_input_tokens": 50}, }, { "model_name": "test", - "litellm_params": {"model": "gpt-4", "api_key": "sk-test"}, + "litellm_params": {"model": "gpt-5.5", "api_key": "sk-test"}, "model_info": {"id": "large", "max_input_tokens": 10000}, }, ], diff --git a/tests/router_unit_tests/test_prompt_management_check.py b/tests/router_unit_tests/test_prompt_management_check.py index 23ad2090e1..81c6c6f013 100644 --- a/tests/router_unit_tests/test_prompt_management_check.py +++ b/tests/router_unit_tests/test_prompt_management_check.py @@ -19,7 +19,7 @@ def test_is_prompt_management_model_optimization(): Optimization: Check if "/" in model name before calling expensive get_model_list(). This short-circuits 99% of requests that use - standard model names like "gpt-4", "claude-3", etc. + standard model names like "gpt-5.5", "claude-3", etc. Tests both negative (early exit) and positive (actual detection) cases. """ @@ -29,17 +29,17 @@ def test_is_prompt_management_model_optimization(): router = Router( model_list=[ { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5"}, }, { "model_name": "claude-3", - "litellm_params": {"model": "anthropic/claude-3-sonnet-20240229"}, + "litellm_params": {"model": "anthropic/claude-sonnet-4-5-20250929"}, }, ] ) - assert router._is_prompt_management_model("gpt-4") is False + assert router._is_prompt_management_model("gpt-5.5") is False assert router._is_prompt_management_model("claude-3") is False # Test 2: Models with "/" but not in model_list -> False after check diff --git a/tests/router_unit_tests/test_router_acancel_batch.py b/tests/router_unit_tests/test_router_acancel_batch.py index 03dd08cd7d..b364a66752 100644 --- a/tests/router_unit_tests/test_router_acancel_batch.py +++ b/tests/router_unit_tests/test_router_acancel_batch.py @@ -21,9 +21,9 @@ def router(): return Router( model_list=[ { - "model_name": "gpt-4", + "model_name": "gpt-5.5", "litellm_params": { - "model": "gpt-4", + "model": "gpt-5.5", "api_key": "fake-key", }, } @@ -44,7 +44,7 @@ async def test_router_acancel_batch(router): # This tests that the router method exists and can be called # The actual API call is mocked response = await router.acancel_batch( - model="gpt-4", + model="gpt-5.5", batch_id="batch_123", ) diff --git a/tests/router_unit_tests/test_router_batch_utils.py b/tests/router_unit_tests/test_router_batch_utils.py index 7334179c65..1b8f713a43 100644 --- a/tests/router_unit_tests/test_router_batch_utils.py +++ b/tests/router_unit_tests/test_router_batch_utils.py @@ -31,11 +31,11 @@ def sample_jsonl_data() -> List[Dict]: return [ { "body": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello"}], } }, - {"body": {"model": "gpt-4", "messages": [{"role": "user", "content": "Hi"}]}}, + {"body": {"model": "gpt-5.5", "messages": [{"role": "user", "content": "Hi"}]}}, ] diff --git a/tests/router_unit_tests/test_router_cooldown_utils.py b/tests/router_unit_tests/test_router_cooldown_utils.py index 33640ad858..ea0cd74d87 100644 --- a/tests/router_unit_tests/test_router_cooldown_utils.py +++ b/tests/router_unit_tests/test_router_cooldown_utils.py @@ -62,8 +62,8 @@ def testing_litellm_router(): return Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_id": "test_deployment", }, { @@ -113,7 +113,7 @@ def test_should_cooldown_deployment_rate_limit_error(testing_litellm_router): """ # Test 429 error (rate limit) -> always cooldown a deployment returning 429s _exception = litellm.exceptions.RateLimitError( - "Rate limit", "openai", "gpt-3.5-turbo" + "Rate limit", "openai", "gpt-5-mini" ) assert ( _should_cooldown_deployment( @@ -129,7 +129,7 @@ def test_should_cooldown_deployment_auth_limit_error(testing_litellm_router): """ # Test 401 error (auth limit) -> always cooldown a deployment returning 401s _exception = litellm.exceptions.AuthenticationError( - "Unauthorized", "openai", "gpt-3.5-turbo" + "Unauthorized", "openai", "gpt-5-mini" ) assert ( _should_cooldown_deployment( @@ -151,7 +151,7 @@ async def test_should_cooldown_deployment(testing_litellm_router): # Test 429 error (rate limit) -> always cooldown a deployment returning 429s _exception = litellm.exceptions.RateLimitError( - "Rate limit", "openai", "gpt-3.5-turbo" + "Rate limit", "openai", "gpt-5-mini" ) assert ( _should_cooldown_deployment( @@ -211,8 +211,8 @@ async def test_should_cooldown_deployment_allowed_fails_set_on_router(): router = Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_id": "test_deployment", }, ] @@ -295,8 +295,8 @@ def router(): return Router( model_list=[ { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5"}, "model_info": { "id": "gpt-4--0", }, @@ -445,7 +445,7 @@ def test_should_cooldown_deployment_minimum_request_threshold(testing_litellm_ro ) _exception = litellm.exceptions.InternalServerError( - "Internal error", "openai", "gpt-3.5-turbo" + "Internal error", "openai", "gpt-5-mini" ) # With only 1 request, should NOT cooldown (below minimum threshold) diff --git a/tests/router_unit_tests/test_router_embedding_headers.py b/tests/router_unit_tests/test_router_embedding_headers.py index 530349a2bc..5bf98243dc 100644 --- a/tests/router_unit_tests/test_router_embedding_headers.py +++ b/tests/router_unit_tests/test_router_embedding_headers.py @@ -32,9 +32,9 @@ class TestRouterEmbeddingHeaders: """ model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -53,12 +53,12 @@ class TestRouterEmbeddingHeaders: data=[{"embedding": [0.1, 0.2, 0.3]}] ) - router.embedding(model="text-embedding-ada-002", input=["test input"]) + router.embedding(model="text-embedding-3-small", input=["test input"]) # Verify _update_kwargs_before_fallbacks was called mock_update.assert_called_once() call_kwargs = mock_update.call_args[1] - assert call_kwargs["model"] == "text-embedding-ada-002" + assert call_kwargs["model"] == "text-embedding-3-small" assert "kwargs" in call_kwargs @pytest.mark.asyncio @@ -70,9 +70,9 @@ class TestRouterEmbeddingHeaders: """ model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -94,13 +94,13 @@ class TestRouterEmbeddingHeaders: ) await router.aembedding( - model="text-embedding-ada-002", input=["test input"] + model="text-embedding-3-small", input=["test input"] ) # Verify _update_kwargs_before_fallbacks was called mock_update.assert_called_once() call_kwargs = mock_update.call_args[1] - assert call_kwargs["model"] == "text-embedding-ada-002" + assert call_kwargs["model"] == "text-embedding-3-small" assert "kwargs" in call_kwargs def test_embedding_propagates_default_litellm_params(self): @@ -114,9 +114,9 @@ class TestRouterEmbeddingHeaders: model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -136,7 +136,7 @@ class TestRouterEmbeddingHeaders: data=[{"embedding": [0.1, 0.2, 0.3]}] ) - router.embedding(model="text-embedding-ada-002", input=["test input"]) + router.embedding(model="text-embedding-3-small", input=["test input"]) # Verify that litellm.embedding was called with the headers mock_litellm_embedding.assert_called_once() @@ -149,7 +149,7 @@ class TestRouterEmbeddingHeaders: # Check that metadata was properly set up assert "metadata" in call_kwargs assert "model_group" in call_kwargs["metadata"] - assert call_kwargs["metadata"]["model_group"] == "text-embedding-ada-002" + assert call_kwargs["metadata"]["model_group"] == "text-embedding-3-small" @pytest.mark.asyncio async def test_aembedding_propagates_default_litellm_params(self): @@ -160,9 +160,9 @@ class TestRouterEmbeddingHeaders: model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -185,7 +185,7 @@ class TestRouterEmbeddingHeaders: ) await router.aembedding( - model="text-embedding-ada-002", input=["test input"] + model="text-embedding-3-small", input=["test input"] ) # Verify that litellm.aembedding was called with the headers @@ -199,7 +199,7 @@ class TestRouterEmbeddingHeaders: # Check that metadata was properly set up assert "metadata" in call_kwargs assert "model_group" in call_kwargs["metadata"] - assert call_kwargs["metadata"]["model_group"] == "text-embedding-ada-002" + assert call_kwargs["metadata"]["model_group"] == "text-embedding-3-small" def test_embedding_metadata_includes_model_group(self): """ @@ -211,7 +211,7 @@ class TestRouterEmbeddingHeaders: { "model_name": "test-embedding-model", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -241,9 +241,9 @@ class TestRouterEmbeddingHeaders: """ model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -257,7 +257,7 @@ class TestRouterEmbeddingHeaders: data=[{"embedding": [0.1, 0.2, 0.3]}] ) - router.embedding(model="text-embedding-ada-002", input=["test input"]) + router.embedding(model="text-embedding-3-small", input=["test input"]) # Verify num_retries was not set in the call (it's handled by function_with_fallbacks) # The important thing is that it was set in kwargs before being passed to function_with_fallbacks @@ -272,9 +272,9 @@ class TestRouterEmbeddingHeaders: """ model_list = [ { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, } @@ -287,7 +287,7 @@ class TestRouterEmbeddingHeaders: data=[{"embedding": [0.1, 0.2, 0.3]}] ) - router.embedding(model="text-embedding-ada-002", input=["test input"]) + router.embedding(model="text-embedding-3-small", input=["test input"]) call_kwargs = mock_litellm_embedding.call_args[1] @@ -306,16 +306,16 @@ class TestRouterEmbeddingHeaders: model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "fake-key", }, }, { - "model_name": "text-embedding-ada-002", + "model_name": "text-embedding-3-small", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fake-key", }, }, @@ -330,7 +330,7 @@ class TestRouterEmbeddingHeaders: mock_completion.return_value = MagicMock() router.completion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": "test"}] + model="gpt-5-mini", messages=[{"role": "user", "content": "test"}] ) completion_kwargs = mock_completion.call_args[1] @@ -341,7 +341,7 @@ class TestRouterEmbeddingHeaders: data=[{"embedding": [0.1, 0.2, 0.3]}] ) - router.embedding(model="text-embedding-ada-002", input=["test input"]) + router.embedding(model="text-embedding-3-small", input=["test input"]) embedding_kwargs = mock_embedding.call_args[1] diff --git a/tests/router_unit_tests/test_router_embedding_integration.py b/tests/router_unit_tests/test_router_embedding_integration.py index 521e1e9399..6f5781336e 100644 --- a/tests/router_unit_tests/test_router_embedding_integration.py +++ b/tests/router_unit_tests/test_router_embedding_integration.py @@ -30,7 +30,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "embedding-deployment-1", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "key-1", "headers": {"X-Deployment": "deployment-1"}, }, @@ -38,7 +38,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "embedding-deployment-2", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "key-2", "headers": {"X-Deployment": "deployment-2"}, }, @@ -75,7 +75,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "test-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "test-key", }, } @@ -117,7 +117,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "test-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "test-key", }, } @@ -170,7 +170,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "test-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "test-key", }, } @@ -194,7 +194,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "test-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "test-key", }, } @@ -222,14 +222,14 @@ class TestRouterEmbeddingIntegration: { "model_name": "shared-embedding-model", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "key-1", }, }, { "model_name": "shared-embedding-model", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "key-2", }, }, @@ -264,14 +264,14 @@ class TestRouterEmbeddingIntegration: { "model_name": "primary-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "primary-key", }, }, { "model_name": "fallback-embedding", "litellm_params": { - "model": "text-embedding-ada-002", + "model": "text-embedding-3-small", "api_key": "fallback-key", }, }, @@ -320,7 +320,7 @@ class TestRouterEmbeddingIntegration: { "model_name": "azure-embedding", "litellm_params": { - "model": "azure/text-embedding-ada-002", + "model": "azure/text-embedding-3-small", "api_key": "azure-key", "api_base": "https://example.openai.azure.com", "api_version": "2024-02-01", diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py index 0ce2dec9b5..3f0afe2a5a 100644 --- a/tests/router_unit_tests/test_router_endpoints.py +++ b/tests/router_unit_tests/test_router_endpoints.py @@ -31,23 +31,23 @@ import asyncio def model_list(): return [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": os.getenv("OPENAI_API_KEY"), }, }, { - "model_name": "gpt-4o", + "model_name": "gpt-5.5", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": os.getenv("OPENAI_API_KEY"), }, }, { - "model_name": "dall-e-3", + "model_name": "gpt-image-1", "litellm_params": { - "model": "dall-e-3", + "model": "gpt-image-1", "api_key": os.getenv("OPENAI_API_KEY"), }, }, @@ -59,9 +59,9 @@ def model_list(): }, }, { - "model_name": "claude-3-5-sonnet-20240620", + "model_name": "claude-sonnet-4-5-20250929", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "mock_response": "hi this is macintosh.", }, }, @@ -323,21 +323,21 @@ async def test_aaaaatext_completion_endpoint(model_list, sync_mode): if sync_mode: response = router.text_completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", prompt="Hello, how are you?", mock_response="I'm fine, thank you!", ) else: ## Test 1: user facing function response = await router.atext_completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", prompt="Hello, how are you?", mock_response="I'm fine, thank you!", ) ## Test 2: underlying function response_2 = await router._atext_completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", prompt="Hello, how are you?", mock_response="I'm fine, thank you!", ) @@ -359,12 +359,12 @@ async def test_router_with_empty_choices(model_list): completion_tokens=10, total_tokens=20, ), - model="gpt-3.5-turbo", + model="gpt-5-mini", object="chat.completion", created=1723081200, ).model_dump() response = await router.acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response=mock_response, ) @@ -1142,7 +1142,7 @@ async def test_init_containers_api_endpoints_managed_id_routes_via_generic_fallb { "model_name": "azure-router-model", "litellm_params": { - "model": "azure/gpt-4", + "model": "azure/gpt-5.5", "api_key": "fake-key", "api_base": "https://westus.api.cognitive.microsoft.com", }, diff --git a/tests/router_unit_tests/test_router_handle_error.py b/tests/router_unit_tests/test_router_handle_error.py index 660b388512..a84c90ccb7 100644 --- a/tests/router_unit_tests/test_router_handle_error.py +++ b/tests/router_unit_tests/test_router_handle_error.py @@ -33,7 +33,7 @@ async def test_send_llm_exception_alert_success(): # Create mock request kwargs request_kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello"}], } @@ -65,7 +65,7 @@ async def test_send_llm_exception_alert_no_logger(): # Create mock request kwargs request_kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello"}], } @@ -94,7 +94,7 @@ async def test_send_llm_exception_alert_when_proxy_server_request_in_kwargs(): # Create mock request kwargs request_kwargs = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello"}], "proxy_server_request": {}, } @@ -145,7 +145,7 @@ async def test_async_raise_no_deployment_exception(): # Call the function result = await async_raise_no_deployment_exception( litellm_router_instance=mock_router, - model="gpt-3.5-turbo", + model="gpt-5-mini", parent_otel_span=None, ) @@ -153,7 +153,7 @@ async def test_async_raise_no_deployment_exception(): assert isinstance(result, RouterRateLimitError) # Assert that the error has the correct properties - assert result.model == "gpt-3.5-turbo" + assert result.model == "gpt-5-mini" assert result.cooldown_time == 30.0 assert result.enable_pre_call_checks is True @@ -166,7 +166,7 @@ async def test_async_raise_no_deployment_exception(): assert isinstance(item, str), f"Expected string ID, got {type(item)}: {item}" # Verify mock calls - mock_router.get_model_ids.assert_called_once_with(model_name="gpt-3.5-turbo") + mock_router.get_model_ids.assert_called_once_with(model_name="gpt-5-mini") mock_router.cooldown_cache.get_min_cooldown.assert_called_once_with( model_ids=["deployment-1", "deployment-2"], parent_otel_span=None ) @@ -241,7 +241,7 @@ async def test_async_raise_no_deployment_exception_none_cooldown_list(): # After the defensive fix, this should handle None gracefully and return empty list result = await async_raise_no_deployment_exception( litellm_router_instance=mock_router, - model="gpt-4", + model="gpt-5.5", parent_otel_span=None, ) @@ -249,7 +249,7 @@ async def test_async_raise_no_deployment_exception_none_cooldown_list(): assert isinstance(result, RouterRateLimitError) # Assert that the error has the correct properties - assert result.model == "gpt-4" + assert result.model == "gpt-5.5" assert result.cooldown_time == 45.0 assert result.enable_pre_call_checks is True diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 59f2d1638e..65d9d6b925 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -21,9 +21,9 @@ from litellm.types.router import Deployment, LiteLLM_Params def model_list(): return [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": os.getenv("OPENAI_API_KEY"), "tpm": 1000, # Add TPM limit so async method doesn't return early "rpm": 100, # Add RPM limit so async method doesn't return early @@ -33,9 +33,9 @@ def model_list(): }, }, { - "model_name": "gpt-4o", + "model_name": "gpt-5.5", "litellm_params": { - "model": "gpt-4o", + "model": "gpt-5.5", "api_key": os.getenv("OPENAI_API_KEY"), }, }, @@ -64,8 +64,8 @@ def model_list(): def test_validate_fallbacks(model_list): - router = Router(model_list=model_list, fallbacks=[{"gpt-4o": "gpt-3.5-turbo"}]) - router.validate_fallbacks(fallback_param=[{"gpt-4o": "gpt-3.5-turbo"}]) + router = Router(model_list=model_list, fallbacks=[{"gpt-5.5": "gpt-5-mini"}]) + router.validate_fallbacks(fallback_param=[{"gpt-5.5": "gpt-5-mini"}]) def test_routing_strategy_init(model_list): @@ -149,9 +149,9 @@ def test_print_deployment(model_list): router = Router(model_list=model_list) deployment = { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": os.getenv("OPENAI_API_KEY"), }, } @@ -190,7 +190,7 @@ def test_completion(model_list): """Test if the completion function is working correctly""" router = Router(model_list=model_list) response = router._completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="I'm fine, thank you!", ) @@ -224,7 +224,7 @@ async def test_router_acompletion_util(model_list): """Test if the underlying '_acompletion' function is working correctly""" router = Router(model_list=model_list) response = await router._acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="I'm fine, thank you!", ) @@ -236,7 +236,7 @@ async def test_router_abatch_completion_one_model_multiple_requests_util(model_l """Test if the 'abatch_completion_one_model_multiple_requests' function is working correctly""" router = Router(model_list=model_list) response = await router.abatch_completion_one_model_multiple_requests( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[ [{"role": "user", "content": "Hello, how are you?"}], [{"role": "user", "content": "Hello, how are you?"}], @@ -253,7 +253,7 @@ async def test_router_schedule_acompletion(model_list): """Test if the 'schedule_acompletion' function is working correctly""" router = Router(model_list=model_list) response = await router.schedule_acompletion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="I'm fine, thank you!", priority=1, @@ -272,7 +272,7 @@ async def test_router_schedule_atext_completion(model_list): ) as mock_atext_completion: mock_atext_completion.return_value = TextCompletionResponse() response = await router.atext_completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", prompt="Hello, how are you?", priority=1, ) @@ -291,9 +291,9 @@ async def test_router_schedule_factory(model_list): ) as mock_atext_completion: mock_atext_completion.return_value = TextCompletionResponse() response = await router._schedule_factory( - model="gpt-3.5-turbo", + model="gpt-5-mini", args=( - "gpt-3.5-turbo", + "gpt-5-mini", "Hello, how are you?", ), priority=1, @@ -310,7 +310,7 @@ async def test_router_function_with_fallbacks(model_list, sync_mode): """Test if the router 'async_function_with_fallbacks' + 'function_with_fallbacks' are working correctly""" router = Router(model_list=model_list) data = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello, how are you?"}], "mock_response": "I'm fine, thank you!", "num_retries": 0, @@ -334,7 +334,7 @@ async def test_router_function_with_retries(model_list, sync_mode): """Test if the router 'async_function_with_retries' + 'function_with_retries' are working correctly""" router = Router(model_list=model_list) data = { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "messages": [{"role": "user", "content": "Hello, how are you?"}], "mock_response": "I'm fine, thank you!", "num_retries": 0, @@ -355,7 +355,7 @@ async def test_router_make_call(model_list): router = Router(model_list=model_list) response = await router.make_call( original_function=router._acompletion, - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="I'm fine, thank you!", ) @@ -364,7 +364,7 @@ async def test_router_make_call(model_list): ## ATEXT_COMPLETION response = await router.make_call( original_function=router._atext_completion, - model="gpt-3.5-turbo", + model="gpt-5-mini", prompt="Hello, how are you?", mock_response="I'm fine, thank you!", ) @@ -373,7 +373,7 @@ async def test_router_make_call(model_list): ## AEMBEDDING response = await router.make_call( original_function=router._aembedding, - model="gpt-3.5-turbo", + model="gpt-5-mini", input="Hello, how are you?", mock_response=[0.1, 0.2, 0.3], ) @@ -394,7 +394,7 @@ def test_update_kwargs_with_deployment(model_list): router = Router(model_list=model_list) kwargs: dict = {"metadata": {}} deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) router._update_kwargs_with_deployment( deployment=deployment, @@ -460,10 +460,10 @@ def test_get_fallback_model_group_from_fallbacks(model_list): """Test if the '_get_fallback_model_group_from_fallbacks' function is working correctly""" router = Router(model_list=model_list) fallback_model_group_name = router._get_fallback_model_group_from_fallbacks( - model_group="gpt-4o", - fallbacks=[{"gpt-4o": "gpt-3.5-turbo"}], + model_group="gpt-5.5", + fallbacks=[{"gpt-5.5": "gpt-5-mini"}], ) - assert fallback_model_group_name == "gpt-3.5-turbo" + assert fallback_model_group_name == "gpt-5-mini" @pytest.mark.parametrize("sync_mode", [True, False]) @@ -474,9 +474,9 @@ async def test_deployment_callback_on_success(sync_mode): model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": os.getenv("OPENAI_API_KEY"), "rpm": 100, }, @@ -486,7 +486,7 @@ async def test_deployment_callback_on_success(sync_mode): router = Router(model_list=model_list) # Get the actual deployment ID that was generated gpt_deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) deployment_id = gpt_deployment["model_info"]["id"] @@ -496,14 +496,14 @@ async def test_deployment_callback_on_success(sync_mode): kwargs = { "litellm_params": { "metadata": { - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", }, "model_info": {"id": deployment_id}, }, "standard_logging_object": standard_logging_payload, } response = litellm.ModelResponse( - model="gpt-3.5-turbo", + model="gpt-5-mini", usage={"total_tokens": 100}, ) if sync_mode: @@ -532,7 +532,7 @@ async def test_deployment_callback_on_failure(model_list): kwargs = { "litellm_params": { "metadata": { - "model_group": "gpt-3.5-turbo", + "model_group": "gpt-5-mini", }, "model_info": {"id": 100}, }, @@ -547,7 +547,7 @@ async def test_deployment_callback_on_failure(model_list): assert result is False model_response = router.completion( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], mock_response="I'm fine, thank you!", ) @@ -575,7 +575,7 @@ def test_deployment_callback_respects_cooldown_time(model_list): kwargs = { "exception": FakeException(), "litellm_params": { - "metadata": {"model_group": "gpt-3.5-turbo"}, + "metadata": {"model_group": "gpt-5-mini"}, "model_info": {"id": 100}, "cooldown_time": 0, }, @@ -610,7 +610,7 @@ def test_update_usage(model_list): """Test if the '_update_usage' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) deployment_id = deployment["model_info"]["id"] request_count = router._update_usage( @@ -635,14 +635,14 @@ def test_should_raise_content_policy_error( """Test if the '_should_raise_content_policy_error' function is working correctly""" router = Router( model_list=model_list, - default_fallbacks=["gpt-4o"] if fallback_type == "default" else None, + default_fallbacks=["gpt-5.5"] if fallback_type == "default" else None, ) assert ( router._should_raise_content_policy_error( - model="gpt-3.5-turbo", + model="gpt-5-mini", response=litellm.ModelResponse( - model="gpt-3.5-turbo", + model="gpt-5-mini", choices=[ { "finish_reason": finish_reason, @@ -653,7 +653,7 @@ def test_should_raise_content_policy_error( ), kwargs={ "content_policy_fallbacks": ( - [{"gpt-3.5-turbo": "gpt-4o"}] + [{"gpt-5-mini": "gpt-5.5"}] if fallback_type == "model-specific" else None ) @@ -667,7 +667,7 @@ def test_get_healthy_deployments(model_list): """Test if the '_get_healthy_deployments' function is working correctly""" router = Router(model_list=model_list) deployments = router._get_healthy_deployments( - model="gpt-3.5-turbo", parent_otel_span=None + model="gpt-5-mini", parent_otel_span=None ) assert len(deployments) > 0 @@ -685,11 +685,11 @@ async def test_routing_strategy_pre_call_checks(model_list, sync_mode): router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) litellm_logging_obj = Logging( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], stream=False, call_type="acompletion", @@ -713,7 +713,7 @@ async def test_routing_strategy_pre_call_checks(model_list, sync_mode): side_effect=litellm.RateLimitError( message="Rate limit error", llm_provider="openai", - model="gpt-3.5-turbo", + model="gpt-5-mini", ) ), ): @@ -752,9 +752,9 @@ def test_create_deployment( os.environ["LITELLM_ENVIRONMENT"] = "staging" deployment = router._create_deployment( deployment_info={}, - _model_name="gpt-3.5-turbo", + _model_name="gpt-5-mini", _litellm_params={ - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test", "custom_llm_provider": "openai", }, @@ -779,7 +779,7 @@ def test_deployment_is_active_for_environment( """Test if the '_deployment_is_active_for_environment' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) if set_supported_environments: os.environ["LITELLM_ENVIRONMENT"] = "staging" @@ -805,7 +805,7 @@ def test_add_deployment(model_list): """Test if the '_add_deployment' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) deployment["model_info"]["id"] = "100" ## Test 1: call user facing function @@ -821,9 +821,9 @@ def test_upsert_deployment(model_list): router = Router(model_list=model_list) print("model list", len(router.model_list)) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) - deployment.litellm_params.model = "gpt-4o" + deployment.litellm_params.model = "gpt-5.5" router.upsert_deployment(deployment=deployment) assert len(router.model_list) == len(model_list) @@ -832,7 +832,7 @@ def test_delete_deployment(model_list): """Test if the 'delete_deployment' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) router.delete_deployment(id=deployment["model_info"]["id"]) assert len(router.model_list) == len(model_list) - 1 @@ -842,7 +842,7 @@ def test_get_model_info(model_list): """Test if the 'get_model_info' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) model_info = router.get_model_info(id=deployment["model_info"]["id"]) assert model_info is not None @@ -852,19 +852,19 @@ def test_get_model_group(model_list): """Test if the 'get_model_group' function is working correctly""" router = Router(model_list=model_list) deployment = router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ) model_group = router.get_model_group(id=deployment["model_info"]["id"]) assert model_group is not None - assert model_group[0]["model_name"] == "gpt-3.5-turbo" + assert model_group[0]["model_name"] == "gpt-5-mini" -@pytest.mark.parametrize("user_facing_model_group_name", ["gpt-3.5-turbo", "gpt-4o"]) +@pytest.mark.parametrize("user_facing_model_group_name", ["gpt-5-mini", "gpt-5.5"]) def test_set_model_group_info(model_list, user_facing_model_group_name): """Test if the 'set_model_group_info' function is working correctly""" router = Router(model_list=model_list) resp = router._set_model_group_info( - model_group="gpt-3.5-turbo", + model_group="gpt-5-mini", user_facing_model_group_name=user_facing_model_group_name, ) assert resp is not None @@ -956,7 +956,7 @@ def test_get_all_deployments(model_list): """Test if the 'get_all_deployments' function is working correctly""" router = Router(model_list=model_list) deployments = router._get_all_deployments( - model_name="gpt-3.5-turbo", model_alias="gpt-3.5-turbo" + model_name="gpt-5-mini", model_alias="gpt-5-mini" ) assert len(deployments) > 0 @@ -981,7 +981,7 @@ def test_common_checks_available_deployment(model_list): """Test if the 'common_checks_available_deployment' function is working correctly""" router = Router(model_list=model_list) _, available_deployments = router._common_checks_available_deployment( - model="gpt-3.5-turbo", + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], input="hi", specific_deployment=False, @@ -994,11 +994,11 @@ def test_filter_cooldown_deployments(model_list): """Test if the 'filter_cooldown_deployments' function is working correctly""" router = Router(model_list=model_list) deployments = router._filter_cooldown_deployments( - healthy_deployments=router._get_all_deployments(model_name="gpt-3.5-turbo"), # type: ignore + healthy_deployments=router._get_all_deployments(model_name="gpt-5-mini"), # type: ignore cooldown_deployments=[], ) assert len(deployments) == len( - router._get_all_deployments(model_name="gpt-3.5-turbo") + router._get_all_deployments(model_name="gpt-5-mini") ) @@ -1009,10 +1009,10 @@ def test_track_deployment_metrics(model_list): router = Router(model_list=model_list) router._track_deployment_metrics( deployment=router.get_deployment_by_model_group_name( - model_group_name="gpt-3.5-turbo" + model_group_name="gpt-5-mini" ), response=ModelResponse( - model="gpt-3.5-turbo", + model="gpt-5-mini", usage={"total_tokens": 100}, ), parent_otel_span=None, @@ -1047,7 +1047,7 @@ def test_get_num_retries_from_retry_policy( print("exception_type", exception_type) calc_num_retries = router.get_num_retries_from_retry_policy( exception=exception_type( - message="test", llm_provider="openai", model="gpt-3.5-turbo" + message="test", llm_provider="openai", model="gpt-5-mini" ) ) assert calc_num_retries == num_retries @@ -1078,7 +1078,7 @@ def test_get_allowed_fails_from_policy( ) calc_allowed_fails = router.get_allowed_fails_from_policy( exception=exception_type( - message="test", llm_provider="openai", model="gpt-3.5-turbo" + message="test", llm_provider="openai", model="gpt-5-mini" ) ) assert calc_allowed_fails == allowed_fails @@ -1170,16 +1170,16 @@ def test_get_model_from_alias(model_list): """Test if the 'get_model_from_alias' function is working correctly""" router = Router( model_list=model_list, - model_group_alias={"gpt-4o": "gpt-3.5-turbo"}, + model_group_alias={"gpt-5.5": "gpt-5-mini"}, ) - model = router._get_model_from_alias(model="gpt-4o") - assert model == "gpt-3.5-turbo" + model = router._get_model_from_alias(model="gpt-5.5") + assert model == "gpt-5-mini" def test_get_deployment_by_litellm_model(model_list): """Test if the 'get_deployment_by_litellm_model' function is working correctly""" router = Router(model_list=model_list) - deployment = router._get_deployment_by_litellm_model(model="gpt-3.5-turbo") + deployment = router._get_deployment_by_litellm_model(model="gpt-5-mini") assert deployment is not None @@ -1239,8 +1239,8 @@ def test_replace_model_in_jsonl(model_list): ( "fo::hi::static::hello", "fo::*::static::*", - "openai/gpt-3.5-turbo", - "openai/gpt-3.5-turbo", + "openai/gpt-5-mini", + "openai/gpt-5-mini", ), ( "bedrock/meta.llama3-70b", @@ -1333,10 +1333,10 @@ async def test_async_callback_filter_deployments(model_list): router = Router(model_list=model_list) - healthy_deployments = router.get_model_list(model_name="gpt-3.5-turbo") + healthy_deployments = router.get_model_list(model_name="gpt-5-mini") new_healthy_deployments = await router.async_callback_filter_deployments( - model="gpt-3.5-turbo", + model="gpt-5-mini", healthy_deployments=healthy_deployments, messages=[], parent_otel_span=None, @@ -1350,10 +1350,10 @@ def test_cached_get_model_group_info(model_list): router = Router(model_list=model_list) # First call - should hit the actual function - result1 = router._cached_get_model_group_info("gpt-3.5-turbo") + result1 = router._cached_get_model_group_info("gpt-5-mini") # Second call with same argument - should hit the cache - result2 = router._cached_get_model_group_info("gpt-3.5-turbo") + result2 = router._cached_get_model_group_info("gpt-5-mini") # Verify results are the same assert result1 == result2 @@ -1437,7 +1437,7 @@ def test_is_auto_router_deployment(model_list): assert router._is_auto_router_deployment(litellm_params_auto) is True # Test case 2: Model doesn't start with "auto_router/" - should return False - litellm_params_regular = LiteLLM_Params(model="gpt-3.5-turbo") + litellm_params_regular = LiteLLM_Params(model="gpt-5-mini") assert router._is_auto_router_deployment(litellm_params_regular) is False # Test case 3: Model is empty string - should return False @@ -1462,8 +1462,8 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list): litellm_params = LiteLLM_Params( model="auto_router/test", auto_router_config_path="/path/to/config", - auto_router_default_model="gpt-3.5-turbo", - auto_router_embedding_model="text-embedding-ada-002", + auto_router_default_model="gpt-5-mini", + auto_router_embedding_model="text-embedding-3-small", ) deployment = Deployment( model_name="test-auto-router", @@ -1479,8 +1479,8 @@ def test_init_auto_router_deployment_success(mock_auto_router, model_list): model_name="test-auto-router", auto_router_config_path="/path/to/config", auto_router_config=None, - default_model="gpt-3.5-turbo", - embedding_model="text-embedding-ada-002", + default_model="gpt-5-mini", + embedding_model="text-embedding-3-small", litellm_router_instance=router, ) @@ -1505,8 +1505,8 @@ def test_init_auto_router_deployment_duplicate_model_name(mock_auto_router, mode litellm_params = LiteLLM_Params( model="auto_router/test", auto_router_config_path="/path/to/config", - auto_router_default_model="gpt-3.5-turbo", - auto_router_embedding_model="text-embedding-ada-002", + auto_router_default_model="gpt-5-mini", + auto_router_embedding_model="text-embedding-3-small", ) deployment = Deployment( model_name="test-auto-router", @@ -1971,7 +1971,7 @@ def test_get_metadata_variable_name_from_kwargs(model_list): # Test case 4: kwargs contains other keys but no metadata keys - should return "metadata" kwargs_other = { - "model": "gpt-4", + "model": "gpt-5.5", "messages": [{"role": "user", "content": "hello"}], } result = router._get_metadata_variable_name_from_kwargs(kwargs_other) @@ -2167,15 +2167,15 @@ def test_get_first_default_fallback(): # Test with default fallback ("*") model_list = [ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "fake-key"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "fake-key"}, } ] - router = Router(model_list=model_list, fallbacks=[{"*": ["gpt-3.5-turbo"]}]) + router = Router(model_list=model_list, fallbacks=[{"*": ["gpt-5-mini"]}]) result = router._get_first_default_fallback() - assert result == "gpt-3.5-turbo" + assert result == "gpt-5-mini" # Test with no fallbacks router_no_fallbacks = Router(model_list=model_list) @@ -2184,7 +2184,7 @@ def test_get_first_default_fallback(): # Test with fallbacks but no default router_no_default = Router( - model_list=model_list, fallbacks=[{"gpt-4": ["gpt-3.5-turbo"]}] + model_list=model_list, fallbacks=[{"gpt-5.5": ["gpt-5-mini"]}] ) result = router_no_default._get_first_default_fallback() assert result is None @@ -2206,16 +2206,16 @@ def test_resolve_model_name_from_model_id(): # Test case 2: model_id directly matches a model_name model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", }, }, ] router = Router(model_list=model_list) - result = router.resolve_model_name_from_model_id("gpt-3.5-turbo") - assert result == "gpt-3.5-turbo" + result = router.resolve_model_name_from_model_id("gpt-5-mini") + assert result == "gpt-5-mini" # Test case 3: model_id matches litellm_params.model exactly model_list = [ @@ -2268,9 +2268,9 @@ def test_resolve_model_name_from_model_id(): # Test case 6: model_id doesn't match anything model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", }, }, @@ -2287,9 +2287,9 @@ def test_resolve_model_name_from_model_id(): # Test case 8: Multiple models, find the correct one model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", }, }, @@ -2309,17 +2309,17 @@ def test_resolve_model_name_from_model_id(): # This tests the has_model_id path in Strategy 1 model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": "test-key", }, }, ] router = Router(model_list=model_list) - result = router.resolve_model_name_from_model_id("gpt-3.5-turbo") - assert result == "gpt-3.5-turbo" + result = router.resolve_model_name_from_model_id("gpt-5-mini") + assert result == "gpt-5-mini" def test_get_valid_args(): @@ -2356,8 +2356,8 @@ def test_get_router_model_info_with_deployment_object(): router = Router( model_list=[ { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4", "api_key": "test-key"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5", "api_key": "test-key"}, "model_info": {"id": "test-id"}, } ] @@ -2373,7 +2373,7 @@ def test_get_router_model_info_with_deployment_object(): # that reuses the existing LiteLLM_Params instead of reconstructing it model_info = router.get_router_model_info( deployment=deployment, - received_model_name="gpt-4", + received_model_name="gpt-5.5", ) # Verify we got valid model info back diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 4371859080..983fc0c4c3 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -22,8 +22,8 @@ class TestRouterIndexManagement: """Test that deleting a deployment updates model_name_to_deployment_indices correctly""" router.model_list = [ {"model_name": "gpt-3.5", "model_info": {"id": "model-1"}}, - {"model_name": "gpt-4", "model_info": {"id": "model-2"}}, - {"model_name": "gpt-4", "model_info": {"id": "model-3"}}, + {"model_name": "gpt-5.5", "model_info": {"id": "model-2"}}, + {"model_name": "gpt-5.5", "model_info": {"id": "model-3"}}, {"model_name": "claude", "model_info": {"id": "model-4"}}, ] router.model_id_to_deployment_index_map = { @@ -34,31 +34,31 @@ class TestRouterIndexManagement: } router.model_name_to_deployment_indices = { "gpt-3.5": [0], - "gpt-4": [1, 2], + "gpt-5.5": [1, 2], "claude": [3], } - # Remove one of the duplicate gpt-4 deployments + # Remove one of the duplicate gpt-5.5 deployments router._update_deployment_indices_after_removal( model_id="model-2", removal_idx=1 ) # Verify indices are shifted correctly assert router.model_name_to_deployment_indices["gpt-3.5"] == [0] - assert router.model_name_to_deployment_indices["gpt-4"] == [ + assert router.model_name_to_deployment_indices["gpt-5.5"] == [ 1 ] # was [1,2], removed 1, shifted 2->1 assert router.model_name_to_deployment_indices["claude"] == [ 2 ] # was [3], shifted to [2] - # Remove the last gpt-4 deployment + # Remove the last gpt-5.5 deployment router._update_deployment_indices_after_removal( model_id="model-3", removal_idx=1 ) - # Verify gpt-4 is removed from dict when no deployments remain - assert "gpt-4" not in router.model_name_to_deployment_indices + # Verify gpt-5.5 is removed from dict when no deployments remain + assert "gpt-5.5" not in router.model_name_to_deployment_indices assert router.model_name_to_deployment_indices["gpt-3.5"] == [0] assert router.model_name_to_deployment_indices["claude"] == [1] @@ -66,13 +66,13 @@ class TestRouterIndexManagement: """Test _build_model_id_to_deployment_index_map function""" model_list = [ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_info": {"id": "model-1"}, }, { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5"}, "model_info": {"id": "model-2"}, }, ] @@ -136,19 +136,19 @@ class TestRouterIndexManagement: "model_info": { "id": "dep-1", "team_id": "team-abc", - "team_public_model_name": "gpt-4o", + "team_public_model_name": "gpt-5.5", }, } router._update_team_model_index(model, 0) - assert router.team_model_to_deployment_indices[("team-abc", "gpt-4o")] == [0] + assert router.team_model_to_deployment_indices[("team-abc", "gpt-5.5")] == [0] router._update_team_model_index(model, 2) - assert router.team_model_to_deployment_indices[("team-abc", "gpt-4o")] == [0, 2] + assert router.team_model_to_deployment_indices[("team-abc", "gpt-5.5")] == [0, 2] router._update_team_model_index( {"model_name": "x", "model_info": {"id": "dep-2"}}, 5 ) assert router.team_model_to_deployment_indices == { - ("team-abc", "gpt-4o"): [0, 2], + ("team-abc", "gpt-5.5"): [0, 2], } def test_has_model_id(self, router): @@ -183,18 +183,18 @@ class TestRouterIndexManagement: """Test _build_model_name_index function""" model_list = [ { - "model_name": "gpt-3.5-turbo", - "litellm_params": {"model": "gpt-3.5-turbo"}, + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini"}, "model_info": {"id": "model-1"}, }, { - "model_name": "gpt-4", - "litellm_params": {"model": "gpt-4"}, + "model_name": "gpt-5.5", + "litellm_params": {"model": "gpt-5.5"}, "model_info": {"id": "model-2"}, }, { - "model_name": "gpt-4", # Duplicate model_name, different deployment - "litellm_params": {"model": "gpt-4"}, + "model_name": "gpt-5.5", # Duplicate model_name, different deployment + "litellm_params": {"model": "gpt-5.5"}, "model_info": {"id": "model-3"}, }, ] @@ -203,14 +203,14 @@ class TestRouterIndexManagement: router._build_model_name_index(model_list) # Verify: model_name_to_deployment_indices is correctly built - assert "gpt-3.5-turbo" in router.model_name_to_deployment_indices - assert "gpt-4" in router.model_name_to_deployment_indices + assert "gpt-5-mini" in router.model_name_to_deployment_indices + assert "gpt-5.5" in router.model_name_to_deployment_indices - # Verify: gpt-3.5-turbo has single deployment - assert router.model_name_to_deployment_indices["gpt-3.5-turbo"] == [0] + # Verify: gpt-5-mini has single deployment + assert router.model_name_to_deployment_indices["gpt-5-mini"] == [0] - # Verify: gpt-4 has multiple deployments - assert router.model_name_to_deployment_indices["gpt-4"] == [1, 2] + # Verify: gpt-5.5 has multiple deployments + assert router.model_name_to_deployment_indices["gpt-5.5"] == [1, 2] # Test: Rebuild index (should clear and rebuild) new_model_list = [ @@ -223,8 +223,8 @@ class TestRouterIndexManagement: router._build_model_name_index(new_model_list) # Verify: Old entries are cleared - assert "gpt-3.5-turbo" not in router.model_name_to_deployment_indices - assert "gpt-4" not in router.model_name_to_deployment_indices + assert "gpt-5-mini" not in router.model_name_to_deployment_indices + assert "gpt-5.5" not in router.model_name_to_deployment_indices # Verify: New entry is added assert "claude-3" in router.model_name_to_deployment_indices diff --git a/tests/router_unit_tests/test_router_prompt_caching.py b/tests/router_unit_tests/test_router_prompt_caching.py index e5ee00e653..574eccda16 100644 --- a/tests/router_unit_tests/test_router_prompt_caching.py +++ b/tests/router_unit_tests/test_router_prompt_caching.py @@ -124,7 +124,7 @@ async def test_router_prompt_caching_same_cacheable_prefix_routes_to_same_deploy { "model_name": "test-model", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_base": "https://exampleopenaiendpoint-production-0ee2.up.railway.app/v1", "api_key": f"test-key-{i}", }, diff --git a/tests/spend_tracking_tests/test_ocr_spend_tracking.py b/tests/spend_tracking_tests/test_ocr_spend_tracking.py index 3c49b696a4..3ce77c5636 100644 --- a/tests/spend_tracking_tests/test_ocr_spend_tracking.py +++ b/tests/spend_tracking_tests/test_ocr_spend_tracking.py @@ -231,7 +231,7 @@ class TestGetLoggingPayloadOCR: def test_non_ocr_call_uses_token_based_usage(self, mock_datetime): """Test that non-OCR calls still use token-based usage""" kwargs = { - "model": "gpt-4", + "model": "gpt-5.5", "call_type": "completion", "litellm_params": {}, "response_cost": 0.02, @@ -240,7 +240,7 @@ class TestGetLoggingPayloadOCR: response_obj = { "id": "completion-test-123", "object": "chat.completion", - "model": "gpt-4", + "model": "gpt-5.5", "usage": { "prompt_tokens": 50, "completion_tokens": 100, diff --git a/tests/spend_tracking_tests/test_spend_accuracy_tests.py b/tests/spend_tracking_tests/test_spend_accuracy_tests.py index b50afeb843..be071f2f0f 100644 --- a/tests/spend_tracking_tests/test_spend_accuracy_tests.py +++ b/tests/spend_tracking_tests/test_spend_accuracy_tests.py @@ -38,7 +38,7 @@ Additional Test Scenarios: # Upstream model the proxy is configured with (spend_tracking_config.yaml). # The proxy computes spend using this model's pricing; the local ground-truth # calculation uses the same pricing table via litellm.cost_per_token. -UPSTREAM_MODEL = "gpt-3.5-turbo" +UPSTREAM_MODEL = "gpt-5-mini" # Batch writer flush cadence in CI is ~2-7s (PROXY_BATCH_WRITE_AT=2 + up to 5s jitter). # Poll every 2s for 60s — plenty of headroom for multiple ticks to land. diff --git a/tests/test_end_users.py b/tests/test_end_users.py index c175bb371e..ff3cc4ec94 100644 --- a/tests/test_end_users.py +++ b/tests/test_end_users.py @@ -180,7 +180,7 @@ async def test_aaaend_user_specific_region(): ## MAKE CALL ## key_gen = await generate_key( - session=session, i=0, models=["gpt-3.5-turbo-end-user-test"] + session=session, i=0, models=["gpt-5-mini-end-user-test"] ) key = key_gen["key"] @@ -190,7 +190,7 @@ async def test_aaaend_user_specific_region(): print("SENDING USER PARAM - {}".format(end_user_obj["user_id"])) result = await client.chat.completions.with_raw_response.create( - model="gpt-3.5-turbo-end-user-test", + model="gpt-5-mini-end-user-test", messages=[{"role": "user", "content": "Hey!"}], user=end_user_obj["user_id"], ) diff --git a/tests/test_litellm_proxy_responses_config.py b/tests/test_litellm_proxy_responses_config.py index 929c2d6c97..0743565874 100644 --- a/tests/test_litellm_proxy_responses_config.py +++ b/tests/test_litellm_proxy_responses_config.py @@ -15,7 +15,7 @@ def test_litellm_proxy_responses_api_config(): ) config = ProviderConfigManager.get_provider_responses_api_config( - model="litellm_proxy/gpt-4", + model="litellm_proxy/gpt-5.5", provider=LlmProviders.LITELLM_PROXY, ) print(f"config: {config}") diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py index 72b8a8cdad..0469ded3f4 100644 --- a/tests/test_ratelimit.py +++ b/tests/test_ratelimit.py @@ -20,9 +20,9 @@ from litellm import utils, Router COMPLETION_TOKENS = 5 base_model_list = [ { - "model_name": "gpt-3.5-turbo", + "model_name": "gpt-5-mini", "litellm_params": { - "model": "gpt-3.5-turbo", + "model": "gpt-5-mini", "api_key": os.getenv("OPENAI_API_KEY"), "max_tokens": COMPLETION_TOKENS, }, @@ -74,14 +74,14 @@ def calculate_limits(list_of_messages): async def async_call(router: Router, list_of_messages) -> Any: tasks = [ - router.acompletion(model="gpt-3.5-turbo", messages=m) for m in list_of_messages + router.acompletion(model="gpt-5-mini", messages=m) for m in list_of_messages ] return await asyncio.gather(*tasks) def sync_call(router: Router, list_of_messages) -> Any: return [ - router.completion(model="gpt-3.5-turbo", messages=m) for m in list_of_messages + router.completion(model="gpt-5-mini", messages=m) for m in list_of_messages ] diff --git a/tests/unified_google_tests/test_litellm_responses_bridge.py b/tests/unified_google_tests/test_litellm_responses_bridge.py index d242b54de1..b2489dfe2a 100644 --- a/tests/unified_google_tests/test_litellm_responses_bridge.py +++ b/tests/unified_google_tests/test_litellm_responses_bridge.py @@ -19,9 +19,9 @@ class TestLiteLLMResponsesBridge(BaseInteractionsTest): """Return the model string for the bridge provider. The bridge provider uses litellm.responses() internally, so we can - use any model that litellm.responses() supports (e.g., gpt-4o). + use any model that litellm.responses() supports (e.g., gpt-5.5). """ - return "gpt-4o" + return "gpt-5.5" def get_api_key(self) -> str: """Return the OpenAI API key from environment."""