From 2b00ea9ee486ee86300a41a2d85df214ab1e86d1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 16 May 2026 07:47:25 +0000 Subject: [PATCH] test(ci): skip Fireworks tests on 404 + Gemini image-size test on 429 Four pre-existing flakes on main that gate this branch's workflow even though they're unrelated to the reasoning_effort_grid suite: 1. tests/local_testing/test_completion.py::test_completion_fireworks_ai 2. tests/local_testing/test_completion_cost.py::test_completion_cost_fireworks_ai[fireworks_ai/llama-v3p3-70b-instruct] 3. tests/llm_translation/test_fireworks_ai_translation.py::test_document_inlining_example[False] The Fireworks-hosted `llama-v3p3-70b-instruct` deployment is currently returning 404 "Model not found, inaccessible, and/or not deployed". These tests pass when the model is deployed; the issue is upstream capacity, not our code path. Wrap the live call in a try/except that pytest.skip's on litellm.NotFoundError so a Fireworks deployment hiccup no longer fails CI for unrelated PRs. 4. tests/llm_translation/test_gemini.py::test_gemini_image_size_limit_exceeded The test fetches the 32MB "Blue Marble 2002" image from Wikimedia to exercise the 50MB image-size cap. CI runners share an IP pool with noisy traffic, so Wikimedia routinely returns HTTP 429. The size-limit check never gets a chance to fire. Catch the 429 BadRequestError and pytest.skip in that case. None of these belong on this PR conceptually, but they're included per request to unblock the workflow before morning. --- .../test_fireworks_ai_translation.py | 23 +++++++------ tests/llm_translation/test_gemini.py | 33 +++++++++++++++---- tests/local_testing/test_completion.py | 2 ++ tests/local_testing/test_completion_cost.py | 5 ++- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/tests/llm_translation/test_fireworks_ai_translation.py b/tests/llm_translation/test_fireworks_ai_translation.py index 1cc6aabdca..164c27ca25 100644 --- a/tests/llm_translation/test_fireworks_ai_translation.py +++ b/tests/llm_translation/test_fireworks_ai_translation.py @@ -118,16 +118,19 @@ def test_document_inlining_example(disable_add_transform_inline_image_block): disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, ) else: - completion = litellm.completion( - model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", - messages=[ - { - "role": "user", - "content": "this is a test request, write a short poem", - }, - ], - disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, - ) + try: + completion = litellm.completion( + model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", + messages=[ + { + "role": "user", + "content": "this is a test request, write a short poem", + }, + ], + disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, + ) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") print(completion) diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 97b0aaee86..85039372af 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1362,8 +1362,12 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) # For Gemini 3, should not force thinkingLevel by default - assert "thinkingLevel" not in result, "Should not force thinkingLevel for Gemini 3" - assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3" + assert ( + "thinkingLevel" not in result + ), "Should not force thinkingLevel for Gemini 3" + assert ( + "thinkingBudget" not in result + ), "Should NOT have thinkingBudget for Gemini 3" assert result["includeThoughts"] is True # Test 2: Anthropic thinking disabled for Gemini 3 @@ -1395,7 +1399,10 @@ def test_anthropic_thinking_param_to_gemini_3_provider_defaults(): ) assert result_zero["includeThoughts"] is False - assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None + assert ( + "thinkingLevel" not in result_zero + or result_zero.get("thinkingLevel") is None + ) # Test 4: Gemini 3 flash-preview should also follow provider defaults by default result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param( @@ -1525,8 +1532,12 @@ def test_anthropic_thinking_param_via_map_openai_params(): # Check that thinkingConfig was created without forced thinkingLevel assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params" thinking_config = result["thinkingConfig"] - assert "thinkingLevel" not in thinking_config, "Should not force thinkingLevel for Gemini 3 by default" - assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3" + assert ( + "thinkingLevel" not in thinking_config + ), "Should not force thinkingLevel for Gemini 3 by default" + assert ( + "thinkingBudget" not in thinking_config + ), "Should NOT have thinkingBudget for Gemini 3" assert thinking_config["includeThoughts"] is True # Test with Gemini 2 model @@ -1614,8 +1625,16 @@ def test_gemini_image_size_limit_exceeded(): } ] - with pytest.raises(litellm.ImageFetchError) as excinfo: - completion(model="gemini/gemini-2.5-flash-lite", messages=messages) + try: + with pytest.raises(litellm.ImageFetchError) as excinfo: + completion(model="gemini/gemini-2.5-flash-lite", messages=messages) + except litellm.BadRequestError as e: + # Wikimedia rate-limits CI runners (HTTP 429) for the Blue Marble + # image, so the size-limit check never gets a chance to fire. Skip + # rather than fail when the upstream host blocks us. + if "429" in str(e) or "Too Many Requests" in str(e): + pytest.skip(f"Wikimedia rate-limited the test fixture image: {e}") + raise error_message = str(excinfo.value) assert "Image size" in error_message diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 6341fa7800..b92251d2d4 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1061,6 +1061,8 @@ def test_completion_fireworks_ai(): messages=messages, ) print(response) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") except Exception as e: pytest.fail(f"Error occurred: {e}") diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 618287e195..d62e4fbd1f 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -1190,7 +1190,10 @@ def test_completion_cost_fireworks_ai(model): litellm.model_cost = litellm.get_model_cost_map(url="") messages = [{"role": "user", "content": "Hey, how's it going?"}] - resp = litellm.completion(model=model, messages=messages) # works fine + try: + resp = litellm.completion(model=model, messages=messages) + except litellm.NotFoundError as e: + pytest.skip(f"Fireworks model unavailable upstream (404): {e}") print(resp) cost = completion_cost(completion_response=resp)