From bf7241abd1cb6d92899df8796fbdb149df01af72 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Mar 2025 18:02:01 -0700 Subject: [PATCH] litellm_proxy_reliability_tests --- .circleci/config.yml | 193 ++++++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 96 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15eacf9f51..61f98405a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -470,102 +470,6 @@ jobs: paths: - litellm_proxy_security_tests_coverage.xml - litellm_proxy_security_tests_coverage - litellm_proxy_reliability_tests: - machine: - image: ubuntu-2204:2023.10.1 - resource_class: xlarge - working_directory: ~/project - steps: - - checkout - - run: - name: Install Docker CLI (In case it's not already installed) - command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io - - run: - name: Install Python 3.9 - command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version - - run: - name: Install Dependencies - command: | - pip install "pytest==7.3.1" - pip install "pytest-asyncio==0.21.1" - pip install aiohttp - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-mock==3.12.0" - pip install "pytest-asyncio==0.21.1" - pip install "assemblyai==0.37.0" - - run: - name: Setup Toxiproxy - command: | - python tests/proxy_reliability_tests/setup_toxi_proxy.py - - run: - name: Build Docker image - command: docker build -t my-app:latest -f ./docker/Dockerfile.database . - - run: - name: Run Docker container - # intentionally give bad redis credentials here - # the OTEL test - should get this as a trace - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=$TOXI_PROXY_DATABASE_URL \ - -e STORE_MODEL_IN_DB="True" \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ - my-app:latest \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug \ - - run: - name: Install curl and dockerize - command: | - sudo apt-get update - sudo apt-get install -y curl - sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz - sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz - sudo rm dockerize-linux-amd64-v0.6.1.tar.gz - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - run: - name: Wait for app to be ready - command: dockerize -wait http://localhost:4000 -timeout 5m - - run: - name: Run tests - command: | - pwd - ls - python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m - - run: - name: Rename the coverage files - command: | - mv coverage.xml litellm_proxy_security_tests_coverage.xml - mv .coverage litellm_proxy_security_tests_coverage - # Store test results - - store_test_results: - path: test-results - - persist_to_workspace: - root: . - paths: - - litellm_proxy_security_tests_coverage.xml - - litellm_proxy_security_tests_coverage litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames docker: - image: cimg/python:3.11 @@ -1984,6 +1888,103 @@ jobs: no_output_timeout: 120m # Clean up first container + litellm_proxy_reliability_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-mock==3.12.0" + pip install "pytest-asyncio==0.21.1" + pip install "assemblyai==0.37.0" + - run: + name: Setup Toxiproxy + command: | + python tests/proxy_reliability_tests/setup_toxi_proxy.py + - run: + name: Build Docker image + command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$TOXI_PROXY_DATABASE_URL \ + -e STORE_MODEL_IN_DB="True" \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: + 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml litellm_proxy_security_tests_coverage.xml + mv .coverage litellm_proxy_security_tests_coverage + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - litellm_proxy_security_tests_coverage.xml + - litellm_proxy_security_tests_coverage + proxy_build_from_pip_tests: # Change from docker to machine executor