diff --git a/.circleci/config.yml b/.circleci/config.yml index eabf4c6129..884ecbee0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -563,39 +563,6 @@ jobs: paths: - realtime_translation_coverage.xml - realtime_translation_coverage - mcp_testing: - docker: - - *python312_image - working_directory: ~/project - - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - uv run --no-sync python -m pytest -vv tests/mcp_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 - no_output_timeout: 15m - - run: - name: Rename the coverage files - command: | - mv coverage.xml mcp_coverage.xml - mv .coverage mcp_coverage - - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - mcp_coverage.xml - - mcp_coverage agent_testing: docker: - *python312_image @@ -794,39 +761,6 @@ jobs: paths: - search_coverage.xml - search_coverage - # Split litellm_mapped_tests into parallel jobs - litellm_mapped_tests_proxy_part1: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run proxy tests part 1 (high-volume directories) - command: | - uv run --no-sync python -m prisma generate - export PYTHONUNBUFFERED=1 - uv run --no-sync python -m pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/client tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part1.xml --durations=10 -n 4 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A - no_output_timeout: 15m - - store_test_results: - path: test-results - litellm_mapped_tests_proxy_part2: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - steps: - - setup_litellm_test_deps - - run: - name: Run proxy tests part 2 (all other tests) - command: | - uv run --no-sync python -m prisma generate - export PYTHONUNBUFFERED=1 - uv run --no-sync python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A - no_output_timeout: 15m - - store_test_results: - path: test-results litellm_mapped_enterprise_tests: docker: - *python312_image @@ -2072,7 +2006,7 @@ jobs: - run: name: Combine Coverage command: | - uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage + uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage uv tool run --from 'coverage[toml]==7.10.6' coverage xml - codecov/upload: file: ./coverage.xml @@ -2407,8 +2341,6 @@ workflows: filters: *main_branches - realtime_translation_testing: filters: *main_branches - - mcp_testing: - filters: *main_branches - agent_testing: filters: *main_branches - guardrails_testing: @@ -2423,10 +2355,6 @@ workflows: filters: *main_branches - litellm_mapped_enterprise_tests: filters: *main_branches - - litellm_mapped_tests_proxy_part1: - filters: *main_branches - - litellm_mapped_tests_proxy_part2: - filters: *main_branches - batches_testing: filters: *main_branches - litellm_utils_testing: @@ -2444,14 +2372,11 @@ workflows: - upload-coverage: requires: - realtime_translation_testing - - mcp_testing - agent_testing - google_generate_content_endpoint_testing - guardrails_testing - ocr_testing - search_testing - - litellm_mapped_tests_proxy_part1 - - litellm_mapped_tests_proxy_part2 - litellm_mapped_enterprise_tests - batches_testing - litellm_utils_testing diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml index 8e0b3568ae..9de3ac3cf5 100644 --- a/.github/workflows/_test-unit-services-base.yml +++ b/.github/workflows/_test-unit-services-base.yml @@ -32,6 +32,11 @@ on: required: false type: boolean default: false + dist: + description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" + required: false + type: string + default: "loadscope" artifact-name: description: "Unique name for the coverage artifact (must be unique per run)" required: false @@ -124,6 +129,7 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} + DIST: ${{ inputs.dist }} DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }} run: | if [ "${WORKERS}" = "0" ]; then @@ -143,7 +149,7 @@ jobs: -n "${WORKERS}" \ --reruns "${RERUNS}" \ --reruns-delay 1 \ - --dist=loadscope \ + --dist="${DIST}" \ --durations=20 \ --cov=litellm \ --cov-report=xml:coverage.xml \ diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index a631a7c300..f8d5bc265a 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -12,6 +12,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Semantic matrix: each shard groups tests by concern (auth, server, logging, …) +# rather than alphabetical letter ranges. Adding a new test file means adding it +# to whichever group it belongs to, not reshuffling slices. +# +# Design targets: +# * Every shard runs in <= 7 minutes on a 4-core runner. +# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop +# conflicts with the logging worker when run in parallel. +# * test_proxy_utils.py runs in its own shard with --dist=worksteal so xdist +# spreads its ~64 functions (many parametrized) across workers instead of +# pinning the whole file to a single worker (the default --dist=loadscope +# behavior for single-file targets). jobs: proxy-db: permissions: @@ -22,26 +34,111 @@ jobs: fail-fast: false matrix: include: - # Key generation tests must NOT run in parallel (event loop conflicts with logging worker) + # Must run serially — event-loop conflict with the logging worker. - test-group: key-generation test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py" workers: 0 - timeout: 30 - - test-group: auth-checks - test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py" - workers: 8 + dist: loadscope timeout: 20 - # test_proxy_utils.py is large (168+ parametrized tests) — run it on its - # own matrix so --dist=loadscope doesn't pin all of it to a single xdist - # worker and push the "remaining" group past the job timeout. + + - test-group: auth-and-jwt + test-path: >- + tests/proxy_unit_tests/test_auth_checks.py + tests/proxy_unit_tests/test_user_api_key_auth.py + tests/proxy_unit_tests/test_jwt.py + tests/proxy_unit_tests/test_jwt_key_mapping.py + tests/proxy_unit_tests/test_proxy_custom_auth.py + tests/proxy_unit_tests/test_key_generate_dynamodb.py + tests/proxy_unit_tests/test_deployed_proxy_keygen.py + workers: 8 + dist: loadscope + timeout: 15 + + # Own shard, --dist=worksteal so parametrized cases fan out across workers. - test-group: proxy-utils test-path: "tests/proxy_unit_tests/test_proxy_utils.py" workers: 8 - timeout: 20 - - test-group: remaining - test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --ignore=tests/proxy_unit_tests/test_proxy_utils.py" + dist: worksteal + timeout: 15 + + - test-group: proxy-server + test-path: >- + tests/proxy_unit_tests/test_proxy_server.py + tests/proxy_unit_tests/test_proxy_server_keys.py + tests/proxy_unit_tests/test_proxy_server_caching.py + tests/proxy_unit_tests/test_proxy_server_langfuse.py + tests/proxy_unit_tests/test_proxy_server_spend.py + tests/proxy_unit_tests/test_aproxy_startup.py + tests/proxy_unit_tests/test_proxy_config_unit_test.py + tests/proxy_unit_tests/test_proxy_routes.py + tests/proxy_unit_tests/test_proxy_gunicorn.py + tests/proxy_unit_tests/test_server_root_path.py + tests/proxy_unit_tests/test_proxy_pass_user_config.py + tests/proxy_unit_tests/test_proxy_token_counter.py workers: 8 - timeout: 30 + dist: loadscope + timeout: 15 + + - test-group: logging-and-callbacks + test-path: >- + tests/proxy_unit_tests/test_custom_callback_input.py + tests/proxy_unit_tests/test_custom_logger_s3_gcs.py + tests/proxy_unit_tests/test_proxy_custom_logger.py + tests/proxy_unit_tests/test_proxy_reject_logging.py + tests/proxy_unit_tests/test_audit_logs_proxy.py + tests/proxy_unit_tests/test_search_api_logging.py + workers: 8 + dist: loadscope + timeout: 15 + + - test-group: db-and-spend + test-path: >- + tests/proxy_unit_tests/test_prisma_client_backoff_retry.py + tests/proxy_unit_tests/test_db_schema_changes.py + tests/proxy_unit_tests/test_db_schema_migration.py + tests/proxy_unit_tests/test_e2e_pod_lock_manager.py + tests/proxy_unit_tests/test_skills_db.py + tests/proxy_unit_tests/test_update_daily_tag_spend.py + tests/proxy_unit_tests/test_update_spend.py + tests/proxy_unit_tests/test_project_endpoints_prisma.py + tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py + workers: 8 + dist: loadscope + timeout: 15 + + - test-group: guardrails-budget-hooks + test-path: >- + tests/proxy_unit_tests/test_proxy_setting_guardrails.py + tests/proxy_unit_tests/test_banned_keyword_list.py + tests/proxy_unit_tests/test_default_end_user_budget_simple.py + tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py + tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py + tests/proxy_unit_tests/test_unit_test_proxy_hooks.py + workers: 8 + dist: loadscope + timeout: 15 + + - test-group: endpoints-and-responses + test-path: >- + tests/proxy_unit_tests/test_blog_posts_endpoint.py + tests/proxy_unit_tests/test_models_fallback_endpoint.py + tests/proxy_unit_tests/test_google_endpoint_routing.py + tests/proxy_unit_tests/test_google_gemini_proxy_request.py + tests/proxy_unit_tests/test_get_favicon.py + tests/proxy_unit_tests/test_get_image.py + tests/proxy_unit_tests/test_ui_path_detection.py + tests/proxy_unit_tests/test_prompt_test_endpoint.py + tests/proxy_unit_tests/test_check_batch_cost.py + tests/proxy_unit_tests/test_check_responses_cost.py + tests/proxy_unit_tests/test_response_polling_handler.py + tests/proxy_unit_tests/test_response_polling_pre_call_checks.py + tests/proxy_unit_tests/test_realtime_cache.py + tests/proxy_unit_tests/test_proxy_exception_mapping.py + tests/proxy_unit_tests/test_custom_tokenizer_bug.py + tests/proxy_unit_tests/test_model_response_typing + workers: 8 + dist: loadscope + timeout: 15 uses: ./.github/workflows/_test-unit-services-base.yml with: test-path: ${{ matrix.test-path }} @@ -49,6 +146,7 @@ jobs: reruns: 2 timeout-minutes: ${{ matrix.timeout }} enable-postgres: true + dist: ${{ matrix.dist }} artifact-name: proxy-db-${{ matrix.test-group }} secrets: DATABASE_URL: ${{ secrets.DATABASE_URL }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index fafc866a3f..1439b2c07f 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -36,6 +36,8 @@ jobs: tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/prompts + tests/test_litellm/proxy/rag_endpoints + tests/test_litellm/proxy/realtime_endpoints tests/test_litellm/proxy/ui_crud_endpoints workers: 2 reruns: 2