diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py index e1165812e2..2ece740bf5 100644 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ b/tests/batches_tests/test_openai_batches_and_files.py @@ -234,9 +234,9 @@ def cleanup_azure_ft_models(): import requests client = AzureOpenAI( - api_key=os.getenv("AZURE_FT_API_KEY"), - azure_endpoint=os.getenv("AZURE_FT_API_BASE"), - api_version=os.getenv("AZURE_API_VERSION"), + api_key=os.getenv("AZURE_AI_API_KEY"), + azure_endpoint=os.getenv("AZURE_AI_API_BASE"), + api_version=os.getenv("AZURE_AI_API_VERSION"), ) _list_ft_jobs = client.fine_tuning.jobs.list() @@ -577,7 +577,10 @@ async def test_vertex_list_batches(monkeypatch): monkeypatch.setattr( "litellm.llms.vertex_ai.batches.handler.VertexAIBatchPrediction._ensure_access_token", - lambda self, credentials, project_id, custom_llm_provider: ("mock-token", "litellm-test-project"), + lambda self, credentials, project_id, custom_llm_provider: ( + "mock-token", + "litellm-test-project", + ), ) with patch( @@ -648,7 +651,7 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error(): async def test_delete_batch_output_file(): """ Test that deleting a batch output file works correctly. - + This test verifies the fix for: - When a batch is retrieved and has an output_file_id, the file object is properly stored - The output file can be deleted without validation errors @@ -656,11 +659,11 @@ async def test_delete_batch_output_file(): """ litellm._turn_on_debug() print("Testing delete batch output file") - + file_name = "openai_batch_completions.jsonl" _current_dir = os.path.dirname(os.path.abspath(__file__)) file_path = os.path.join(_current_dir, file_name) - + # Create file for batch file_obj = await litellm.acreate_file( file=open(file_path, "rb"), @@ -669,7 +672,7 @@ async def test_delete_batch_output_file(): ) print("Response from creating file=", file_obj) batch_input_file_id = file_obj.id - + # Create batch create_batch_response = await litellm.acreate_batch( completion_window="24h", @@ -678,36 +681,37 @@ async def test_delete_batch_output_file(): custom_llm_provider="openai", ) print("Batch created with ID=", create_batch_response.id) - + # Retrieve batch to get output_file_id retrieved_batch = await litellm.aretrieve_batch( - batch_id=create_batch_response.id, - custom_llm_provider="openai" + batch_id=create_batch_response.id, custom_llm_provider="openai" ) print("Retrieved batch=", retrieved_batch) - + # If batch has completed and has output file, test deleting it if retrieved_batch.output_file_id: print(f"Testing deletion of output file: {retrieved_batch.output_file_id}") - + # This is the key test - deleting the output file should work # without validation errors (file_object should not be None) delete_output_file_response = await litellm.afile_delete( - file_id=retrieved_batch.output_file_id, - custom_llm_provider="openai" + file_id=retrieved_batch.output_file_id, custom_llm_provider="openai" ) - + print("Delete output file response=", delete_output_file_response) assert delete_output_file_response.id == retrieved_batch.output_file_id - assert delete_output_file_response.deleted is True or hasattr(delete_output_file_response, 'id') + assert delete_output_file_response.deleted is True or hasattr( + delete_output_file_response, "id" + ) print("✓ Successfully deleted batch output file") else: - print("⚠ Batch has not completed yet or no output file available, skipping output file deletion test") - + print( + "⚠ Batch has not completed yet or no output file available, skipping output file deletion test" + ) + # Clean up - delete the input file delete_input_file_response = await litellm.afile_delete( - file_id=batch_input_file_id, - custom_llm_provider="openai" + file_id=batch_input_file_id, custom_llm_provider="openai" ) print("Delete input file response=", delete_input_file_response) assert delete_input_file_response.id == batch_input_file_id diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index f87f2005d3..d014b3f72a 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -246,35 +246,6 @@ async def test_audio_transcription_health_check(): print(response) -@pytest.mark.asyncio -@pytest.mark.parametrize( - "model", ["azure/gpt-4o-realtime-preview", "openai/gpt-4o-realtime-preview"] -) -async def test_async_realtime_health_check(model, mocker): - """ - Test Health Check with Valid models passes - - """ - mock_websocket = AsyncMock() - mock_connect = AsyncMock().__aenter__.return_value = mock_websocket - mocker.patch("websockets.connect", return_value=mock_connect) - - litellm.set_verbose = True - model_params = { - "model": model, - } - if model == "azure/gpt-4o-realtime-preview": - model_params["api_base"] = os.getenv("AZURE_REALTIME_API_BASE") - model_params["api_key"] = os.getenv("AZURE_REALTIME_API_KEY") - model_params["api_version"] = os.getenv("AZURE_REALTIME_API_VERSION") - response = await litellm.ahealth_check( - model_params=model_params, - mode="realtime", - ) - print(response) - assert response == {} - - def test_update_litellm_params_for_health_check(): """ Test if _update_litellm_params_for_health_check correctly: diff --git a/tests/llm_translation/test_azure_ai.py b/tests/llm_translation/test_azure_ai.py index 823fad7648..b6f53e4e95 100644 --- a/tests/llm_translation/test_azure_ai.py +++ b/tests/llm_translation/test_azure_ai.py @@ -188,35 +188,6 @@ def test_azure_ai_services_with_api_version(): ) -@pytest.mark.skip(reason="Skipping due to cohere ssl issues") -def test_completion_azure_ai_command_r(): - try: - import os - - litellm.set_verbose = True - - os.environ["AZURE_AI_API_BASE"] = os.getenv("AZURE_COHERE_API_BASE", "") - os.environ["AZURE_AI_API_KEY"] = os.getenv("AZURE_COHERE_API_KEY", "") - - response = completion( - model="azure_ai/command-r-plus", - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is the meaning of life?"} - ], - } - ], - ) # type: ignore - - assert "azure_ai" in response.model - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - def test_azure_deepseek_reasoning_content(): import json @@ -387,7 +358,7 @@ async def test_azure_ai_model_router(): response = await litellm.acompletion( model="azure_ai/model_router/azure-model-router", messages=[{"role": "user", "content": "hi who is this"}], - api_base="https://ishaa-mh6uutut-swedencentral.cognitiveservices.azure.com/openai/v1/", + api_base=os.getenv("AZURE_MODEL_ROUTER_API_BASE"), api_key=os.getenv("AZURE_MODEL_ROUTER_API_KEY"), ) print("response: ", response) diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index e68d271a11..f7885fb8a0 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -126,7 +126,9 @@ async def test_router_provider_wildcard_routing(): print("response 3 = ", response3) response4 = await router.acompletion( - model=os.environ.get("CI_CD_DEFAULT_ANTHROPIC_MODEL", "claude-haiku-4-5-20251001"), + model=os.environ.get( + "CI_CD_DEFAULT_ANTHROPIC_MODEL", "claude-haiku-4-5-20251001" + ), messages=[{"role": "user", "content": "hello"}], ) @@ -356,51 +358,6 @@ async def test_router_retries(sync_mode): print(response.choices[0].message) -@pytest.mark.parametrize( - "mistral_api_base", - [ - "os.environ/AZURE_MISTRAL_API_BASE", - "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com/v1/", - "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com/v1", - "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com/", - "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com", - ], -) -@pytest.mark.skip( - reason="Router no longer creates clients, this is delegated to the provider integration." -) -def test_router_azure_ai_studio_init(mistral_api_base): - router = Router( - model_list=[ - { - "model_name": "test-model", - "litellm_params": { - "model": "azure/mistral-large-latest", - "api_key": "os.environ/AZURE_MISTRAL_API_KEY", - "api_base": mistral_api_base, - }, - "model_info": {"id": 1234}, - } - ] - ) - - # model_client = router._get_client( - # deployment={"model_info": {"id": 1234}}, client_type="sync_client", kwargs={} - # ) - # url = getattr(model_client, "_base_url") - # uri_reference = str(getattr(url, "_uri_reference")) - - # print(f"uri_reference: {uri_reference}") - - # assert "/v1/" in uri_reference - # assert uri_reference.count("v1") == 1 - response = router.completion( - model="azure/mistral-large-latest", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - ) - assert response is not None - - def test_exception_raising(): # this tests if the router raises an exception when invalid params are set # in this test both deployments have bad keys - Keep this test. It validates if the router raises the most recent exception @@ -409,8 +366,8 @@ def test_exception_raising(): try: print("testing if router raises an exception") - old_api_key = os.environ["AZURE_API_KEY"] - os.environ["AZURE_API_KEY"] = "" + old_api_key = os.environ["AZURE_AI_API_KEY"] + os.environ["AZURE_AI_API_KEY"] = "" model_list = [ { "model_name": "gpt-3.5-turbo", # openai model name @@ -418,7 +375,7 @@ def test_exception_raising(): "model": "azure/gpt-4.1-mini", "api_key": "bad-key", "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), }, "tpm": 240000, "rpm": 1800, @@ -446,16 +403,16 @@ def test_exception_raising(): model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hello this request will fail"}], ) - os.environ["AZURE_API_KEY"] = old_api_key + os.environ["AZURE_AI_API_KEY"] = old_api_key pytest.fail(f"Should have raised an Auth Error") except openai.AuthenticationError: print( "Test Passed: Caught an OPENAI AUTH Error, Good job. This is what we needed!" ) - os.environ["AZURE_API_KEY"] = old_api_key + os.environ["AZURE_AI_API_KEY"] = old_api_key router.reset() except Exception as e: - os.environ["AZURE_API_KEY"] = old_api_key + os.environ["AZURE_AI_API_KEY"] = old_api_key print("Got unexpected exception on router!", e) @@ -530,7 +487,7 @@ def test_call_one_endpoint(): # this test makes a completion calls azure/gpt-4.1-mini, it should work try: print("Testing calling a specific deployment") - old_api_key = os.environ["AZURE_API_KEY"] + old_api_key = os.environ["AZURE_AI_API_KEY"] model_list = [ { @@ -539,7 +496,7 @@ def test_call_one_endpoint(): "model": "azure/gpt-4.1-mini", "api_key": old_api_key, "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), }, "tpm": 240000, "rpm": 1800, @@ -548,8 +505,8 @@ def test_call_one_endpoint(): "model_name": "text-embedding-ada-002", "litellm_params": { "model": "azure/text-embedding-ada-002", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], + "api_key": os.environ["AZURE_AI_API_KEY"], + "api_base": os.environ["AZURE_AI_API_BASE"], }, "tpm": 100000, "rpm": 10000, @@ -562,7 +519,7 @@ def test_call_one_endpoint(): set_verbose=True, num_retries=1, ) # type: ignore - old_api_base = os.environ.pop("AZURE_API_BASE", None) + old_api_base = os.environ.pop("AZURE_AI_API_BASE", None) async def call_azure_completion(): response = await router.acompletion( @@ -584,8 +541,8 @@ def test_call_one_endpoint(): asyncio.run(call_azure_completion()) asyncio.run(call_azure_embedding()) - os.environ["AZURE_API_BASE"] = old_api_base - os.environ["AZURE_API_KEY"] = old_api_key + os.environ["AZURE_AI_API_BASE"] = old_api_base + os.environ["AZURE_AI_API_KEY"] = old_api_key except Exception as e: print(f"FAILED TEST") pytest.fail(f"Got unexpected exception on router! - {e}") @@ -594,7 +551,6 @@ def test_call_one_endpoint(): # test_call_one_endpoint() - @pytest.mark.asyncio @pytest.mark.parametrize("sync_mode", [True, False]) async def test_async_router_context_window_fallback(sync_mode): @@ -708,9 +664,9 @@ def test_router_context_window_check_pre_call_check_in_group_custom_model_info() "model_name": "gpt-3.5-turbo", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_API_KEY"), + "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), "base_model": "azure/gpt-35-turbo", "mock_response": "Hello world 1!", }, @@ -762,9 +718,9 @@ def test_router_context_window_check_pre_call_check(): "model_name": "gpt-3.5-turbo", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_API_KEY"), + "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), "base_model": "azure/gpt-35-turbo", "mock_response": "Hello world 1!", }, @@ -816,9 +772,9 @@ def test_router_context_window_check_pre_call_check_out_group(): "model_name": "gpt-3.5-turbo-small", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_API_KEY"), + "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), "base_model": "azure/gpt-35-turbo", }, }, @@ -896,9 +852,9 @@ def test_router_region_pre_call_check(allowed_model_region): "model_name": "gpt-3.5-turbo", # openai model name "litellm_params": { # params for litellm completion/embedding call "model": "azure/gpt-4.1-mini", - "api_key": os.getenv("AZURE_API_KEY"), + "api_key": os.getenv("AZURE_AI_API_KEY"), "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), + "api_base": os.getenv("AZURE_AI_API_BASE"), "base_model": "azure/gpt-35-turbo", "region_name": allowed_model_region, }, @@ -1173,8 +1129,8 @@ def test_azure_embedding_on_router(): "model_name": "text-embedding-ada-002", "litellm_params": { "model": "azure/text-embedding-ada-002", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], + "api_key": os.environ["AZURE_AI_API_KEY"], + "api_base": os.environ["AZURE_AI_API_BASE"], }, "tpm": 100000, "rpm": 10000, @@ -1381,8 +1337,8 @@ def test_reading_keys_os_environ(): "model_name": "gpt-3.5-turbo", "litellm_params": { "model": "gpt-3.5-turbo", - "api_key": "os.environ/AZURE_API_KEY", - "api_base": "os.environ/AZURE_API_BASE", + "api_key": "os.environ/AZURE_AI_API_KEY", + "api_base": "os.environ/AZURE_AI_API_BASE", "api_version": "os.environ/AZURE_API_VERSION", "timeout": "os.environ/AZURE_TIMEOUT", "stream_timeout": "os.environ/AZURE_STREAM_TIMEOUT", @@ -1394,11 +1350,11 @@ def test_reading_keys_os_environ(): router = Router(model_list=model_list) for model in router.model_list: assert ( - model["litellm_params"]["api_key"] == os.environ["AZURE_API_KEY"] - ), f"{model['litellm_params']['api_key']} vs {os.environ['AZURE_API_KEY']}" + model["litellm_params"]["api_key"] == os.environ["AZURE_AI_API_KEY"] + ), f"{model['litellm_params']['api_key']} vs {os.environ['AZURE_AI_API_KEY']}" assert ( - model["litellm_params"]["api_base"] == os.environ["AZURE_API_BASE"] - ), f"{model['litellm_params']['api_base']} vs {os.environ['AZURE_API_BASE']}" + model["litellm_params"]["api_base"] == os.environ["AZURE_AI_API_BASE"] + ), f"{model['litellm_params']['api_base']} vs {os.environ['AZURE_AI_API_BASE']}" assert ( model["litellm_params"]["api_version"] == os.environ["AZURE_API_VERSION"] @@ -1415,8 +1371,8 @@ def test_reading_keys_os_environ(): print("passed testing of reading keys from os.environ") model_id = model["model_info"]["id"] async_client: openai.AsyncAzureOpenAI = router.cache.get_cache(f"{model_id}_async_client") # type: ignore - assert async_client.api_key == os.environ["AZURE_API_KEY"] - assert async_client.base_url == os.environ["AZURE_API_BASE"] + assert async_client.api_key == os.environ["AZURE_AI_API_KEY"] + assert async_client.base_url == os.environ["AZURE_AI_API_BASE"] assert async_client.max_retries == int( os.environ["AZURE_MAX_RETRIES"] ), f"{async_client.max_retries} vs {os.environ['AZURE_MAX_RETRIES']}" @@ -1428,8 +1384,8 @@ def test_reading_keys_os_environ(): print("\n Testing async streaming client") stream_async_client: openai.AsyncAzureOpenAI = router.cache.get_cache(f"{model_id}_stream_async_client") # type: ignore - assert stream_async_client.api_key == os.environ["AZURE_API_KEY"] - assert stream_async_client.base_url == os.environ["AZURE_API_BASE"] + assert stream_async_client.api_key == os.environ["AZURE_AI_API_KEY"] + assert stream_async_client.base_url == os.environ["AZURE_AI_API_BASE"] assert stream_async_client.max_retries == int( os.environ["AZURE_MAX_RETRIES"] ), f"{stream_async_client.max_retries} vs {os.environ['AZURE_MAX_RETRIES']}" @@ -1440,8 +1396,8 @@ def test_reading_keys_os_environ(): print("\n Testing sync client") client: openai.AzureOpenAI = router.cache.get_cache(f"{model_id}_client") # type: ignore - assert client.api_key == os.environ["AZURE_API_KEY"] - assert client.base_url == os.environ["AZURE_API_BASE"] + assert client.api_key == os.environ["AZURE_AI_API_KEY"] + assert client.base_url == os.environ["AZURE_AI_API_BASE"] assert client.max_retries == int( os.environ["AZURE_MAX_RETRIES"] ), f"{client.max_retries} vs {os.environ['AZURE_MAX_RETRIES']}" @@ -1452,8 +1408,8 @@ def test_reading_keys_os_environ(): print("\n Testing sync stream client") stream_client: openai.AzureOpenAI = router.cache.get_cache(f"{model_id}_stream_client") # type: ignore - assert stream_client.api_key == os.environ["AZURE_API_KEY"] - assert stream_client.base_url == os.environ["AZURE_API_BASE"] + assert stream_client.api_key == os.environ["AZURE_AI_API_KEY"] + assert stream_client.base_url == os.environ["AZURE_AI_API_BASE"] assert stream_client.max_retries == int( os.environ["AZURE_MAX_RETRIES"] ), f"{stream_client.max_retries} vs {os.environ['AZURE_MAX_RETRIES']}" @@ -1503,7 +1459,7 @@ def test_reading_openai_keys_os_environ(): for model in router.model_list: assert ( model["litellm_params"]["api_key"] == os.environ["OPENAI_API_KEY"] - ), f"{model['litellm_params']['api_key']} vs {os.environ['AZURE_API_KEY']}" + ), f"{model['litellm_params']['api_key']} vs {os.environ['AZURE_AI_API_KEY']}" assert float(model["litellm_params"]["timeout"]) == float( os.environ["AZURE_TIMEOUT"] ), f"{model['litellm_params']['timeout']} vs {os.environ['AZURE_TIMEOUT']}" @@ -1574,7 +1530,9 @@ def test_router_anthropic_key_dynamic(): { "model_name": "anthropic-claude", "litellm_params": { - "model": os.environ.get("CI_CD_DEFAULT_ANTHROPIC_MODEL", "claude-haiku-4-5-20251001"), + "model": os.environ.get( + "CI_CD_DEFAULT_ANTHROPIC_MODEL", "claude-haiku-4-5-20251001" + ), "api_key": anthropic_api_key, }, } @@ -2273,8 +2231,8 @@ async def test_router_batch_endpoints(provider): "model_name": "my-custom-name", "litellm_params": { "model": "azure/gpt-4o-mini", - "api_base": os.getenv("AZURE_API_BASE"), - "api_key": os.getenv("AZURE_API_KEY"), + "api_base": os.getenv("AZURE_AI_API_BASE"), + "api_key": os.getenv("AZURE_AI_API_KEY"), }, }, ] @@ -2452,8 +2410,8 @@ def test_is_team_specific_model(): # "model_name": "gpt-3.5-turbo", # "litellm_params": { # "model": "azure/gpt-4.1-mini", -# "api_key": os.getenv("AZURE_API_KEY"), -# "api_base": os.getenv("AZURE_API_BASE"), +# "api_key": os.getenv("AZURE_AI_API_KEY"), +# "api_base": os.getenv("AZURE_AI_API_BASE"), # "tpm": 100000, # "rpm": 100000, # }, @@ -2462,8 +2420,8 @@ def test_is_team_specific_model(): # "model_name": "gpt-3.5-turbo", # "litellm_params": { # "model": "azure/gpt-4.1-mini", -# "api_key": os.getenv("AZURE_API_KEY"), -# "api_base": os.getenv("AZURE_API_BASE"), +# "api_key": os.getenv("AZURE_AI_API_KEY"), +# "api_base": os.getenv("AZURE_AI_API_BASE"), # "tpm": 500, # "rpm": 500, # },