diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b369477c2..2a534b7973 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,10 +208,12 @@ jobs: name: Run tests command: | mkdir test-results + # Discover test files TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ - --split-by=filesize \ + --split-by=timings \ --verbose \ --command="xargs python -m pytest \ -vv \ @@ -222,7 +224,7 @@ jobs: -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \ -n 4 \ --timeout=300 \ - --timeout_method=thread" + no_output_timeout: 120m - run: name: Rename the coverage files @@ -509,7 +511,6 @@ jobs: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - parallelism: 4 steps: - checkout - setup_google_dns @@ -531,21 +532,9 @@ jobs: - run: name: Run tests command: | - mkdir test-results - # Find test files only in local_testing - TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --split-by=filesize \ - --verbose \ - --command="xargs python -m pytest -o junit_family=legacy \ - -k 'router' \ - --cov=litellm \ - --cov-report=xml \ - -n 4 \ - --dist=loadscope \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -vv" + pwd + ls + python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m - run: name: Rename the coverage files @@ -1816,6 +1805,7 @@ jobs: pip install "mlflow==2.17.2" pip install "anthropic==0.52.0" pip install "blockbuster==1.5.24" + pip install "pytest-xdist==3.6.1" # Run pytest and generate JUnit XML report - setup_litellm_enterprise_pip - run: @@ -1823,7 +1813,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m - run: name: Rename the coverage files diff --git a/tests/local_testing/test_add_update_models.py b/tests/local_testing/test_add_update_models.py index 6d7ae597b5..4b5ec95d3f 100644 --- a/tests/local_testing/test_add_update_models.py +++ b/tests/local_testing/test_add_update_models.py @@ -16,6 +16,8 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest, logging, asyncio import litellm +import litellm.proxy +import litellm.proxy.proxy_server from litellm.proxy.management_endpoints.model_management_endpoints import ( add_new_model, update_model, diff --git a/tests/test_litellm/integrations/arize/test_arize_utils.py b/tests/test_litellm/integrations/arize/test_arize_utils.py index 6eb6e67a08..9a9f3d5afc 100644 --- a/tests/test_litellm/integrations/arize/test_arize_utils.py +++ b/tests/test_litellm/integrations/arize/test_arize_utils.py @@ -84,7 +84,7 @@ def test_arize_set_attributes(): ArizeLogger.set_arize_attributes(span, kwargs, response_obj) # Validate that the expected number of attributes were set - assert span.set_attribute.call_count == 28 + assert span.set_attribute.call_count == 26 # Metadata attached to the span span.set_attribute.assert_any_call( @@ -108,7 +108,8 @@ def test_arize_set_attributes(): # Response metadata span.set_attribute.assert_any_call("llm.response.id", "chatcmpl-ID") span.set_attribute.assert_any_call("llm.response.model", "gpt-4o") - span.set_attribute.assert_any_call(SpanAttributes.OPENINFERENCE_SPAN_KIND, "LLM") + # Span kind is set to TOOL when tools are present + span.set_attribute.assert_any_call(SpanAttributes.OPENINFERENCE_SPAN_KIND, "TOOL") # Request message content and metadata span.set_attribute.assert_any_call( @@ -125,14 +126,14 @@ def test_arize_set_attributes(): # Tool call definitions and function names span.set_attribute.assert_any_call( - f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_NAME}", "get_weather" + f"{SpanAttributes.LLM_TOOLS}.0.name", "get_weather" ) span.set_attribute.assert_any_call( - f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_DESCRIPTION}", + f"{SpanAttributes.LLM_TOOLS}.0.description", "Fetches weather details.", ) span.set_attribute.assert_any_call( - f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_PARAMETERS}", + f"{SpanAttributes.LLM_TOOLS}.0.parameters", json.dumps( { "type": "object", @@ -144,16 +145,6 @@ def test_arize_set_attributes(): ), ) - # Tool calls captured from optional_params - span.set_attribute.assert_any_call( - f"{MessageAttributes.MESSAGE_TOOL_CALLS}.0.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", - "get_weather", - ) - span.set_attribute.assert_any_call( - f"{MessageAttributes.MESSAGE_TOOL_CALLS}.1.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", - "get_stock_price", - ) - # Invocation parameters span.set_attribute.assert_any_call( SpanAttributes.LLM_INVOCATION_PARAMETERS, '{"user": "test_user"}'