From a4bfdf242729141744cd91bbe6c5a45dbd5c15b6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 20 Jan 2026 09:37:43 +0530 Subject: [PATCH] Fix: total timeout is not respected --- litellm/main.py | 25 +++++++- tests/local_testing/test_timeout.py | 91 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index ae27b4145b..09868cb794 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -599,7 +599,16 @@ async def acompletion( ctx = contextvars.copy_context() func_with_context = partial(ctx.run, func) - init_response = await loop.run_in_executor(None, func_with_context) + # Wrap with timeout if specified + if timeout is not None: + timeout_value = float(timeout) if not isinstance(timeout, (int, float)) else timeout + init_response = await asyncio.wait_for( + loop.run_in_executor(None, func_with_context), + timeout=timeout_value + ) + else: + init_response = await loop.run_in_executor(None, func_with_context) + if isinstance(init_response, dict) or isinstance( init_response, ModelResponse ): ## CACHING SCENARIO @@ -607,7 +616,11 @@ async def acompletion( response = ModelResponse(**init_response) response = init_response elif asyncio.iscoroutine(init_response): - response = await init_response + if timeout is not None: + timeout_value = float(timeout) if not isinstance(timeout, (int, float)) else timeout + response = await asyncio.wait_for(init_response, timeout=timeout_value) + else: + response = await init_response else: response = init_response # type: ignore @@ -624,6 +637,14 @@ async def acompletion( loop=loop ) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls) return response + except asyncio.TimeoutError as e: + custom_llm_provider = custom_llm_provider or "openai" + from litellm.exceptions import Timeout + raise Timeout( + message=f"Request timed out after {timeout} seconds", + model=model, + llm_provider=custom_llm_provider, + ) except Exception as e: custom_llm_provider = custom_llm_provider or "openai" raise exception_type( diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index 7f9837e81d..bb6dbc938b 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -285,3 +285,94 @@ async def test_anthropic_timeout(streaming, sync_mode): ) print(type(e)) pass + + +@pytest.mark.asyncio +async def test_timeout_respects_total_time_not_per_retry(): + """ + Test that timeout applies to the TOTAL operation time, not per-retry. + + This test ensures that when a user sets timeout=2, the entire operation + (including all retries) times out at ~2 seconds, not at 2s * num_retries. + + This is a regression test for the issue where timeout was being applied + per-retry attempt, causing the total time to be much longer than expected. + """ + litellm.set_verbose = False + + timeout_value = 2.0 + # Allow for some overhead (network, processing, etc.) + # but ensure we don't wait for multiple retries + max_allowed_time = timeout_value + 1.0 # 3 seconds max + + start_time = time.time() + + try: + # This should timeout because we're asking for a long response + # with a very short timeout + response = await litellm.acompletion( + model="gpt-3.5-turbo", + timeout=timeout_value, + messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}], + ) + pytest.fail("Expected timeout error but got a response") + except (openai.APITimeoutError, litellm.exceptions.Timeout) as e: + elapsed_time = time.time() - start_time + + print(f"Timeout occurred after {elapsed_time:.2f} seconds") + print(f"Expected timeout: {timeout_value} seconds") + print(f"Max allowed time: {max_allowed_time} seconds") + + # Verify that the timeout happened within the expected time window + # It should be close to timeout_value, not timeout_value * num_retries + assert elapsed_time < max_allowed_time, ( + f"Timeout took too long! Expected ~{timeout_value}s, " + f"got {elapsed_time:.2f}s. This suggests timeout is being " + f"applied per-retry instead of to the total operation." + ) + + # Also verify it's not TOO fast (sanity check) + assert elapsed_time >= timeout_value * 0.5, ( + f"Timeout happened too quickly: {elapsed_time:.2f}s. " + f"Expected at least {timeout_value * 0.5}s" + ) + + print("✓ Timeout correctly applied to total operation time, not per-retry") + except Exception as e: + pytest.fail( + f"Expected timeout error but got different error: {type(e).__name__}: {e}" + ) + + +@pytest.mark.asyncio +async def test_timeout_with_retries_disabled(): + """ + Test that timeout works correctly when retries are explicitly disabled. + This should timeout even faster since there are no retry attempts. + """ + litellm.set_verbose = False + + timeout_value = 2.0 + max_allowed_time = timeout_value + 0.5 # Even tighter bound with no retries + + start_time = time.time() + + try: + response = await litellm.acompletion( + model="gpt-3.5-turbo", + timeout=timeout_value, + max_retries=0, # Disable retries + messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}], + ) + pytest.fail("Expected timeout error but got a response") + except (openai.APITimeoutError, litellm.exceptions.Timeout) as e: + elapsed_time = time.time() - start_time + + print(f"Timeout with no retries occurred after {elapsed_time:.2f} seconds") + + assert elapsed_time < max_allowed_time, ( + f"Timeout took too long even with retries disabled! " + f"Expected ~{timeout_value}s, got {elapsed_time:.2f}s" + ) + + print("✓ Timeout works correctly with retries disabled")