diff --git a/.circleci/config.yml b/.circleci/config.yml index 6259c18c8b..003cda9a3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,39 @@ commands: cd enterprise python -m pip install -e . cd .. + setup_litellm_test_deps: + steps: + - checkout + - setup_google_dns + - restore_cache: + keys: + - v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} + - v2-litellm-deps- + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest-mock==3.12.0" + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "hypercorn==0.17.3" + pip install "pydantic==2.10.2" + pip install "mcp==1.10.1" + pip install "requests-mock>=1.12.1" + pip install "responses==0.25.7" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install "semantic_router==0.1.10" + pip install "fastapi-offline==1.7.3" + - setup_litellm_enterprise_pip + - save_cache: + paths: + - ~/.cache/pip + key: v2-litellm-deps-{{ checksum "requirements.txt" }}-{{ checksum ".circleci/config.yml" }} jobs: # Add Windows testing job @@ -1128,60 +1161,88 @@ jobs: paths: - search_coverage.xml - search_coverage - litellm_mapped_tests: + # Split litellm_mapped_tests into 3 parallel jobs for 3x faster execution + litellm_mapped_tests_proxy: docker: - image: cimg/python:3.11 auth: username: ${DOCKERHUB_USERNAME} password: ${DOCKERHUB_PASSWORD} working_directory: ~/project - + resource_class: xlarge steps: - - checkout - - setup_google_dns + - setup_litellm_test_deps - run: - name: Install Dependencies + name: Run proxy tests command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest-mock==3.12.0" - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-cov==5.0.0" - pip install "pytest-asyncio==0.21.1" - pip install "respx==0.22.0" - pip install "hypercorn==0.17.3" - pip install "pydantic==2.10.2" - pip install "mcp==1.10.1" - pip install "requests-mock>=1.12.1" - pip install "responses==0.25.7" - pip install "pytest-xdist==3.6.1" - pip install "pytest-forked==1.6.0" - pip install "semantic_router==0.1.10" - pip install "fastapi-offline==1.7.3" - - setup_litellm_enterprise_pip - # Run pytest and generate JUnit XML report - - run: - name: Run litellm tests - command: | - pwd - ls - python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8 + python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING no_output_timeout: 120m - run: name: Rename the coverage files command: | - mv coverage.xml litellm_mapped_tests_coverage.xml - mv .coverage litellm_mapped_tests_coverage - - # Store test results + mv coverage.xml litellm_proxy_tests_coverage.xml + mv .coverage litellm_proxy_tests_coverage - store_test_results: path: test-results - persist_to_workspace: root: . paths: - - litellm_mapped_tests_coverage.xml - - litellm_mapped_tests_coverage + - litellm_proxy_tests_coverage.xml + - litellm_proxy_tests_coverage + litellm_mapped_tests_llms: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run LLM provider tests + command: | + python -m pytest tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-llms.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_llms_tests_coverage.xml + mv .coverage litellm_llms_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_llms_tests_coverage.xml + - litellm_llms_tests_coverage + litellm_mapped_tests_core: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + resource_class: xlarge + steps: + - setup_litellm_test_deps + - run: + name: Run core tests + command: | + python -m pytest tests/test_litellm --ignore=tests/test_litellm/proxy --ignore=tests/test_litellm/llms --cov=litellm --cov-report=xml --junitxml=test-results/junit-core.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_core_tests_coverage.xml + mv .coverage litellm_core_tests_coverage + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_core_tests_coverage.xml + - litellm_core_tests_coverage litellm_mapped_enterprise_tests: docker: - image: cimg/python:3.11 @@ -2709,6 +2770,9 @@ jobs: - run: name: Install Dependencies command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pip install "pytest==7.3.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" @@ -2851,6 +2915,9 @@ jobs: - run: name: Run tests command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pwd ls python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 @@ -2881,7 +2948,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage + coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_proxy_tests_coverage litellm_llms_tests_coverage litellm_core_tests_coverage litellm_mapped_tests_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -3447,7 +3514,19 @@ workflows: only: - main - /litellm_.*/ - - litellm_mapped_tests: + - litellm_mapped_tests_proxy: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_llms: + filters: + branches: + only: + - main + - /litellm_.*/ + - litellm_mapped_tests_core: filters: branches: only: @@ -3498,7 +3577,9 @@ workflows: - llm_responses_api_testing - ocr_testing - search_testing - - litellm_mapped_tests + - litellm_mapped_tests_proxy + - litellm_mapped_tests_llms + - litellm_mapped_tests_core - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing @@ -3563,7 +3644,9 @@ workflows: - llm_responses_api_testing - ocr_testing - search_testing - - litellm_mapped_tests + - litellm_mapped_tests_proxy + - litellm_mapped_tests_llms + - litellm_mapped_tests_core - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing