[Ci/CD] Speed up litellm mapped tests job on circle ci (#16688)

* fix mapped tests speed

* fixes

* split up test_litellm

* refactor mapped tests

.circleci/config.yml

@@ -24,6 +24,39 @@ commands:
             cd enterprise
             python -m pip install -e .
             cd ..
+  setup_litellm_test_deps:
+    steps:
+      - checkout
+      - setup_google_dns
+      - restore_cache:
+          keys:
+            - v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
+            - v2-litellm-deps-
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest-mock==3.12.0"
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+            pip install "hypercorn==0.17.3"
+            pip install "pydantic==2.10.2"
+            pip install "mcp==1.10.1"
+            pip install "requests-mock>=1.12.1"
+            pip install "responses==0.25.7"
+            pip install "pytest-xdist==3.6.1"
+            pip install "pytest-timeout==2.2.0"
+            pip install "semantic_router==0.1.10"
+            pip install "fastapi-offline==1.7.3"
+      - setup_litellm_enterprise_pip
+      - save_cache:
+          paths:
+            - ~/.cache/pip
+          key: v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }}
 
 jobs:
   # Add Windows testing job
@@ -1128,60 +1161,88 @@ jobs:
           paths:
             - search_coverage.xml
             - search_coverage
-  litellm_mapped_tests:
+  # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution
+  litellm_mapped_tests_proxy:
     docker:
       - image: cimg/python:3.11
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
     working_directory: ~/project
-
+    resource_class: xlarge
     steps:
-      - checkout
-      - setup_google_dns
+      - setup_litellm_test_deps
       - run:
-          name: Install Dependencies
+          name: Run proxy tests
           command: |
-            python -m pip install --upgrade pip
-            python -m pip install -r requirements.txt
-            pip install "pytest-mock==3.12.0"
-            pip install "pytest==7.3.1"
-            pip install "pytest-retry==1.6.3"
-            pip install "pytest-cov==5.0.0"
-            pip install "pytest-asyncio==0.21.1"
-            pip install "respx==0.22.0"
-            pip install "hypercorn==0.17.3"
-            pip install "pydantic==2.10.2"
-            pip install "mcp==1.10.1"
-            pip install "requests-mock>=1.12.1"
-            pip install "responses==0.25.7"
-            pip install "pytest-xdist==3.6.1"
-            pip install "pytest-forked==1.6.0"
-            pip install "semantic_router==0.1.10"
-            pip install "fastapi-offline==1.7.3"
-      - setup_litellm_enterprise_pip
-      # Run pytest and generate JUnit XML report
-      - run:
-          name: Run litellm tests
-          command: |
-            pwd
-            ls
-            python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
+            python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
           command: |
-            mv coverage.xml litellm_mapped_tests_coverage.xml
-            mv .coverage litellm_mapped_tests_coverage
-
-      # Store test results
+            mv coverage.xml litellm_proxy_tests_coverage.xml
+            mv .coverage litellm_proxy_tests_coverage
       - store_test_results:
           path: test-results
       - persist_to_workspace:
           root: .
           paths:
-            - litellm_mapped_tests_coverage.xml
-            - litellm_mapped_tests_coverage
+            - litellm_proxy_tests_coverage.xml
+            - litellm_proxy_tests_coverage
+  litellm_mapped_tests_llms:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+    resource_class: xlarge
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Run LLM provider tests
+          command: |
+            python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_llms_tests_coverage.xml
+            mv .coverage litellm_llms_tests_coverage
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_llms_tests_coverage.xml
+            - litellm_llms_tests_coverage
+  litellm_mapped_tests_core:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+    resource_class: xlarge
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Run core tests
+          command: |
+            python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_core_tests_coverage.xml
+            mv .coverage litellm_core_tests_coverage
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_core_tests_coverage.xml
+            - litellm_core_tests_coverage
   litellm_mapped_enterprise_tests:
     docker:
       - image: cimg/python:3.11
@@ -2709,6 +2770,9 @@ jobs:
       - run:
           name: Install Dependencies
           command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
@@ -2851,6 +2915,9 @@ jobs:
       - run:
           name: Run tests
           command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
             pwd
             ls
             python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
@@ -2881,7 +2948,7 @@ jobs:
             python -m venv venv
             . venv/bin/activate
             pip install coverage
-            coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
+            coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_proxy_tests_coverage litellm_llms_tests_coverage litellm_core_tests_coverage litellm_mapped_tests_coverage
             coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -3447,7 +3514,19 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
-      - litellm_mapped_tests:
+      - litellm_mapped_tests_proxy:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_mapped_tests_llms:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - litellm_mapped_tests_core:
           filters:
             branches:
               only:
@@ -3498,7 +3577,9 @@ workflows:
             - llm_responses_api_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_tests
+            - litellm_mapped_tests_proxy
+            - litellm_mapped_tests_llms
+            - litellm_mapped_tests_core
             - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
@@ -3563,7 +3644,9 @@ workflows:
             - llm_responses_api_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_tests
+            - litellm_mapped_tests_proxy
+            - litellm_mapped_tests_llms
+            - litellm_mapped_tests_core
             - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing