mirror of
https://github.com/tiennm99/litellm.git
synced 2026-06-17 22:48:35 +00:00
Yuneng Jiang
95e1babf67
TogetherAIConfig.get_supported_openai_params called get_model_info(), whose first line calls litellm.get_supported_openai_params() — which for together_ai routes straight back into this method. The recursion only terminated when Python's recursion limit was hit or when _get_model_info_helper raised "not mapped" at the deepest level. Either way the try/except caught it, so the bug stayed silent — but the cycle ran ~332 deep every time, emitting hundreds of DEBUG log lines per call. Surfaced as "infinite loop" in CI when the success_handler thread emitted that log spam against an already-closed stderr during test teardown. Replace the get_model_info() call with supports_function_calling(), which uses _get_model_info_helper directly and does not call get_supported_openai_params. Measured drop from 332 to 2 _get_model_info_helper calls per first uncached lookup. Also swap the test model from Qwen/Qwen3.5-9B (not in model_cost map) back to a mapped serverless model, Qwen/Qwen2.5-7B-Instruct-Turbo. The mapping gap is what made the recursion's tail end raise up into the success handler during teardown in the first place.
55 lines
1.6 KiB
Python
55 lines
1.6 KiB
Python
#### What this tests ####
|
|
# This tests error handling + logging (esp. for sentry breadcrumbs)
|
|
|
|
import sys, os
|
|
import traceback
|
|
|
|
sys.path.insert(
|
|
0, os.path.abspath("../..")
|
|
) # Adds the parent directory to the system path
|
|
import pytest
|
|
import litellm
|
|
from litellm import completion
|
|
|
|
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
|
|
|
## All your mistral deployments ##
|
|
model_list = [
|
|
{
|
|
"model_name": "mistral-7b-instruct",
|
|
"litellm_params": { # params for litellm completion/embedding call
|
|
"model": "replicate/mistralai/mistral-7b-instruct-v0.1:83b6a56e7c828e667f21fd596c338fd4f0039b46bcfa18d973e8e70e455fda70",
|
|
"api_key": os.getenv("REPLICATE_API_KEY"),
|
|
},
|
|
},
|
|
{
|
|
"model_name": "mistral-7b-instruct",
|
|
"litellm_params": { # params for litellm completion/embedding call
|
|
"model": "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo",
|
|
"api_key": os.getenv("TOGETHERAI_API_KEY"),
|
|
},
|
|
},
|
|
{
|
|
"model_name": "mistral-7b-instruct",
|
|
"litellm_params": {
|
|
"model": "deepinfra/mistralai/Mistral-7B-Instruct-v0.1",
|
|
"api_key": os.getenv("DEEPINFRA_API_KEY"),
|
|
},
|
|
},
|
|
]
|
|
|
|
|
|
def test_multiple_deployments():
|
|
try:
|
|
## LiteLLM completion call ## returns first response
|
|
response = completion(
|
|
model="mistral-7b-instruct", messages=messages, model_list=model_list
|
|
)
|
|
print(f"response: {response}")
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
pytest.fail(f"An exception occurred: {e}")
|
|
|
|
|
|
test_multiple_deployments()
|