Merge pull request #19695 from BerriAI/litellm_ci_fix_yj_03

[Infra] CI/CD - Fixing failing tests

.circleci/config.yml

@@ -208,10 +208,12 @@ jobs:
           name: Run tests
           command: |
             mkdir test-results
+
             # Discover test files
             TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+
             echo "$TEST_FILES" | circleci tests run \
-              --split-by=filesize \
+              --split-by=timings \
               --verbose \
               --command="xargs python -m pytest \
                 -vv \
@@ -222,7 +224,7 @@ jobs:
                 -k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \
                 -n 4 \
                 --timeout=300 \
-                --timeout_method=thread"
+
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -509,7 +511,6 @@ jobs:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
     working_directory: ~/project
-    parallelism: 4
     steps:
       - checkout
       - setup_google_dns
@@ -531,21 +532,9 @@ jobs:
       - run:
           name: Run tests
           command: |
-            mkdir test-results
-            # Find test files only in local_testing
-            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --split-by=filesize \
-              --verbose \
-              --command="xargs python -m pytest -o junit_family=legacy \
-                -k 'router' \
-                --cov=litellm \
-                --cov-report=xml \
-                -n 4 \
-                --dist=loadscope \
-                --junitxml=test-results/junit.xml \
-                --durations=5 \
-                -vv"
+            pwd
+            ls
+            python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -1816,6 +1805,7 @@ jobs:
             pip install "mlflow==2.17.2"
             pip install "anthropic==0.52.0"
             pip install "blockbuster==1.5.24"
+            pip install "pytest-xdist==3.6.1"
       # Run pytest and generate JUnit XML report
       - setup_litellm_enterprise_pip
       - run:
@@ -1823,7 +1813,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest -vv tests/logging_callback_tests --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest -vv tests/logging_callback_tests --cov=litellm -n 4 --cov-report=xml -s -v --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files

tests/local_testing/test_add_update_models.py

@@ -16,6 +16,8 @@ sys.path.insert(
 )  # Adds the parent directory to the system path
 import pytest, logging, asyncio
 import litellm
+import litellm.proxy
+import litellm.proxy.proxy_server
 from litellm.proxy.management_endpoints.model_management_endpoints import (
     add_new_model,
     update_model,

tests/test_litellm/integrations/arize/test_arize_utils.py

@@ -84,7 +84,7 @@ def test_arize_set_attributes():
     ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
 
     # Validate that the expected number of attributes were set
-    assert span.set_attribute.call_count == 28
+    assert span.set_attribute.call_count == 26
 
     # Metadata attached to the span
     span.set_attribute.assert_any_call(
@@ -108,7 +108,8 @@ def test_arize_set_attributes():
     # Response metadata
     span.set_attribute.assert_any_call("llm.response.id", "chatcmpl-ID")
     span.set_attribute.assert_any_call("llm.response.model", "gpt-4o")
-    span.set_attribute.assert_any_call(SpanAttributes.OPENINFERENCE_SPAN_KIND, "LLM")
+    # Span kind is set to TOOL when tools are present
+    span.set_attribute.assert_any_call(SpanAttributes.OPENINFERENCE_SPAN_KIND, "TOOL")
 
     # Request message content and metadata
     span.set_attribute.assert_any_call(
@@ -125,14 +126,14 @@ def test_arize_set_attributes():
 
     # Tool call definitions and function names
     span.set_attribute.assert_any_call(
-        f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_NAME}", "get_weather"
+        f"{SpanAttributes.LLM_TOOLS}.0.name", "get_weather"
     )
     span.set_attribute.assert_any_call(
-        f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_DESCRIPTION}",
+        f"{SpanAttributes.LLM_TOOLS}.0.description",
         "Fetches weather details.",
     )
     span.set_attribute.assert_any_call(
-        f"{SpanAttributes.LLM_TOOLS}.0.{SpanAttributes.TOOL_PARAMETERS}",
+        f"{SpanAttributes.LLM_TOOLS}.0.parameters",
         json.dumps(
             {
                 "type": "object",
@@ -144,16 +145,6 @@ def test_arize_set_attributes():
         ),
     )
 
-    # Tool calls captured from optional_params
-    span.set_attribute.assert_any_call(
-        f"{MessageAttributes.MESSAGE_TOOL_CALLS}.0.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}",
-        "get_weather",
-    )
-    span.set_attribute.assert_any_call(
-        f"{MessageAttributes.MESSAGE_TOOL_CALLS}.1.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}",
-        "get_stock_price",
-    )
-
     # Invocation parameters
     span.set_attribute.assert_any_call(
         SpanAttributes.LLM_INVOCATION_PARAMETERS, '{"user": "test_user"}'