mirror of
https://github.com/tiennm99/litellm.git
synced 2026-07-03 11:18:50 +00:00
Fix flaky batch tests: mock vertex auth and skip on DNS failure
- test_avertex_batch_prediction: Add google.auth.default mock and env vars so the test doesn't depend on real GCP credentials (was already a unit test with mocked HTTP, just missing auth mock) - test_async_create_batch[openai]: Add DNS pre-check that skips gracefully when api.openai.com is unreachable instead of failing after 4 retries Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -29,10 +29,26 @@ verbose_logger.setLevel(logging.DEBUG)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
import random
|
||||
import socket
|
||||
import httpx
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
|
||||
def _can_resolve_openai():
|
||||
"""Check if api.openai.com is reachable (DNS resolves)."""
|
||||
try:
|
||||
socket.getaddrinfo("api.openai.com", 443, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
||||
return True
|
||||
except socket.gaierror:
|
||||
return False
|
||||
|
||||
|
||||
skip_if_no_openai_network = pytest.mark.skipif(
|
||||
not _can_resolve_openai(),
|
||||
reason="Cannot resolve api.openai.com - skipping integration test due to DNS issues",
|
||||
)
|
||||
|
||||
|
||||
def load_vertex_ai_credentials():
|
||||
# Define the path to the vertex_key.json file
|
||||
print("loading vertex ai credentials")
|
||||
@@ -78,6 +94,7 @@ def load_vertex_ai_credentials():
|
||||
|
||||
@pytest.mark.parametrize("provider", ["openai"]) # , "azure"
|
||||
@pytest.mark.asyncio
|
||||
@skip_if_no_openai_network
|
||||
async def test_create_batch(provider):
|
||||
"""
|
||||
1. Create File for Batch completion
|
||||
@@ -252,6 +269,7 @@ def cleanup_azure_ft_models():
|
||||
@pytest.mark.parametrize("provider", ["openai"])
|
||||
@pytest.mark.asyncio()
|
||||
@pytest.mark.flaky(retries=3, delay=1)
|
||||
@skip_if_no_openai_network
|
||||
async def test_async_create_batch(provider):
|
||||
"""
|
||||
1. Create File for Batch completion
|
||||
@@ -464,9 +482,24 @@ mock_vertex_list_response = {
|
||||
@pytest.mark.asyncio
|
||||
async def test_avertex_batch_prediction(monkeypatch):
|
||||
monkeypatch.setenv("GCS_BUCKET_NAME", "litellm-local")
|
||||
monkeypatch.setenv("VERTEXAI_PROJECT", "mock-project")
|
||||
monkeypatch.setenv("VERTEXAI_LOCATION", "us-central1")
|
||||
|
||||
# Mock Google auth so the test doesn't need real credentials
|
||||
mock_creds = MagicMock()
|
||||
mock_creds.token = "mock-token"
|
||||
mock_creds.valid = True
|
||||
mock_creds.expiry = None
|
||||
monkeypatch.setattr(
|
||||
"google.auth.default",
|
||||
lambda *args, **kwargs: (mock_creds, "mock-project"),
|
||||
)
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
client = AsyncHTTPHandler()
|
||||
# Configure mock response object
|
||||
mock_response = MagicMock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
async def mock_side_effect(*args, **kwargs):
|
||||
print("args", args, "kwargs", kwargs)
|
||||
@@ -478,21 +511,10 @@ async def test_avertex_batch_prediction(monkeypatch):
|
||||
mock_response.status_code = 200
|
||||
return mock_response
|
||||
|
||||
with patch.object(
|
||||
client, "post", side_effect=mock_side_effect
|
||||
) as mock_post, patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
||||
side_effect=mock_side_effect,
|
||||
) as mock_global_post:
|
||||
# Configure mock responses
|
||||
mock_response = MagicMock()
|
||||
mock_response.raise_for_status.return_value = None
|
||||
|
||||
# Set up different responses for different API calls
|
||||
|
||||
mock_post.side_effect = mock_side_effect
|
||||
mock_global_post.side_effect = mock_side_effect
|
||||
|
||||
# load_vertex_ai_credentials()
|
||||
litellm.set_verbose = True
|
||||
litellm._turn_on_debug()
|
||||
file_name = "vertex_batch_completions.jsonl"
|
||||
@@ -504,7 +526,6 @@ async def test_avertex_batch_prediction(monkeypatch):
|
||||
file=open(file_path, "rb"),
|
||||
purpose="batch",
|
||||
custom_llm_provider="vertex_ai",
|
||||
client=client
|
||||
)
|
||||
print("Response from creating file=", file_obj)
|
||||
|
||||
@@ -623,6 +644,7 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@skip_if_no_openai_network
|
||||
async def test_delete_batch_output_file():
|
||||
"""
|
||||
Test that deleting a batch output file works correctly.
|
||||
|
||||
Reference in New Issue
Block a user