From e7b047223e1725be573d879e348e2cb2e7563bcd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 20:35:11 -0700 Subject: [PATCH 1/9] add langsmith logging test --- tests/load_tests/test_langsmith_load_test.py | 136 +++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 tests/load_tests/test_langsmith_load_test.py diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py new file mode 100644 index 0000000000..ba241c7f45 --- /dev/null +++ b/tests/load_tests/test_langsmith_load_test.py @@ -0,0 +1,136 @@ +import sys + +import os + +sys.path.insert(0, os.path.abspath("../..")) + +import asyncio +import litellm +from litellm._logging import verbose_logger +import logging +import time +import pytest + + +def test_langsmith_logging_async(): + # this tests time added to make langsmith logging calls, vs just acompletion calls + try: + + os.environ["LANGSMITH_API_KEY"] = "lsv2_anything" + os.environ["LANGSMITH_PROJECT"] = "pr-b" + os.environ["LANGSMITH_BASE_URL"] = "http://0.0.0.0:8090" + + litellm.set_verbose = True + verbose_logger.setLevel(logging.DEBUG) + + # Make 5 calls with an empty success_callback + litellm.success_callback = [] + litellm.callbacks = [] + litellm._async_success_callback = [] + litellm._async_failure_callback = [] + litellm._async_failure_callback = [] + litellm.failure_callback = [] + start_time_empty_callback = asyncio.run(make_async_calls()) + print("done with no callback test") + + print("starting langsmith test") + # Make 5 calls with success_callback set to "langsmith" + litellm.success_callback = ["langsmith"] + start_time_langsmith = asyncio.run(make_async_calls()) + print("done with langsmith test") + + # Compare the time for both scenarios + print(f"Time taken with success_callback='langsmith': {start_time_langsmith}") + print(f"Time taken with empty success_callback: {start_time_empty_callback}") + + # Calculate the percentage difference + percentage_diff = ( + abs(start_time_langsmith - start_time_empty_callback) + / start_time_empty_callback + * 100 + ) + + # Assert that the difference is not more than 10% + assert ( + percentage_diff < 10 + ), f"Performance difference of {percentage_diff:.2f}% exceeds 10% threshold" + + print(f"Performance difference: {percentage_diff:.2f}%") + + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"An exception occurred - {e}") + + +async def make_async_calls(metadata=None, **completion_kwargs): + tasks = [] + for _ in range(100): + tasks.append(create_async_task()) + + # Measure the start time before running the tasks + start_time = asyncio.get_event_loop().time() + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + + # Print the responses when tasks return + for idx, response in enumerate(responses): + print(f"Response from Task {idx + 1}: {response}") + + await asyncio.sleep(1) + + for _ in range(100): + tasks.append(create_async_task()) + + # Measure the start time before running the tasks + start_time = asyncio.get_event_loop().time() + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + + # Print the responses when tasks return + for idx, response in enumerate(responses): + print(f"Response from Task {idx + 1}: {response}") + + await asyncio.sleep(1) + + for _ in range(100): + tasks.append(create_async_task()) + + # Measure the start time before running the tasks + start_time = asyncio.get_event_loop().time() + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + + # Print the responses when tasks return + for idx, response in enumerate(responses): + print(f"Response from Task {idx + 1}: {response}") + + # Calculate the total time taken + total_time = asyncio.get_event_loop().time() - start_time + + return total_time + + +def create_async_task(**completion_kwargs): + """ + Creates an async task for the litellm.acompletion function. + This is just the task, but it is not run here. + To run the task it must be awaited or used in other asyncio coroutine execution functions like asyncio.gather. + Any kwargs passed to this function will be passed to the litellm.acompletion function. + By default a standard set of arguments are used for the litellm.acompletion function. + """ + completion_args = { + "model": "openai/chatgpt-v-2", + "api_version": "2024-02-01", + "messages": [{"role": "user", "content": "This is a test"}], + "max_tokens": 5, + "temperature": 0.7, + "timeout": 5, + "user": "langfuse_latency_test_user", + "mock_response": "hello from my load test", + } + completion_args.update(completion_kwargs) + return asyncio.create_task(litellm.acompletion(**completion_args)) From 850b5dbadc64d5947bb760d3e20626f53cd0ffc4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 20:47:12 -0700 Subject: [PATCH 2/9] add otel load test --- tests/load_tests/test_otel_load_test.py | 108 ++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/load_tests/test_otel_load_test.py diff --git a/tests/load_tests/test_otel_load_test.py b/tests/load_tests/test_otel_load_test.py new file mode 100644 index 0000000000..24cabaa093 --- /dev/null +++ b/tests/load_tests/test_otel_load_test.py @@ -0,0 +1,108 @@ +import sys + +import os + +sys.path.insert(0, os.path.abspath("../..")) + +import asyncio +import litellm +from litellm._logging import verbose_logger +import logging +import time +import pytest + + +def test_otel_logging_async(): + # this tests time added to make otel logging calls, vs just acompletion calls + try: + + os.environ["OTEL_EXPORTER"] = "otlp_http" + os.environ["OTEL_ENDPOINT"] = "http://0.0.0.0:8090/traces" + os.environ["OTEL_HEADERS"] = "Authorization=K0BSwd" + + # Make 5 calls with an empty success_callback + litellm.success_callback = [] + litellm.callbacks = [] + litellm._async_success_callback = [] + litellm._async_failure_callback = [] + litellm._async_failure_callback = [] + litellm.failure_callback = [] + start_time_empty_callback = asyncio.run(make_async_calls()) + print("done with no callback test") + + print("starting otel test") + # Make 5 calls with success_callback set to "otel" + litellm.callbacks = ["otel"] + start_time_otel = asyncio.run(make_async_calls()) + print("done with otel test") + + # Compare the time for both scenarios + print(f"Time taken with success_callback='otel': {start_time_otel}") + print(f"Time taken with empty success_callback: {start_time_empty_callback}") + + # Calculate the percentage difference + percentage_diff = ( + abs(start_time_otel - start_time_empty_callback) + / start_time_empty_callback + * 100 + ) + + # Assert that the difference is not more than 10% + assert ( + percentage_diff < 10 + ), f"Performance difference of {percentage_diff:.2f}% exceeds 10% threshold" + + print(f"Performance difference: {percentage_diff:.2f}%") + + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"An exception occurred - {e}") + + +async def make_async_calls(metadata=None, **completion_kwargs): + total_start_time = asyncio.get_event_loop().time() + tasks = [] + + async def create_and_run_task(): + task = create_async_task(**completion_kwargs) + response = await task + print(f"Response: {response}") + + for _ in range(3): # Run for 10 seconds + # Create 100 tasks + tasks = [] + for _ in range(100): + tasks.append(asyncio.create_task(create_and_run_task())) + + # Wait for any remaining tasks to complete + await asyncio.gather(*tasks) + + await asyncio.sleep(1) + + # Calculate the total time taken + total_time = asyncio.get_event_loop().time() - total_start_time + + return total_time + + +def create_async_task(**completion_kwargs): + """ + Creates an async task for the litellm.acompletion function. + This is just the task, but it is not run here. + To run the task it must be awaited or used in other asyncio coroutine execution functions like asyncio.gather. + Any kwargs passed to this function will be passed to the litellm.acompletion function. + By default a standard set of arguments are used for the litellm.acompletion function. + """ + completion_args = { + "model": "openai/chatgpt-v-2", + "api_version": "2024-02-01", + "messages": [{"role": "user", "content": "This is a test" * 100}], + "max_tokens": 5, + "temperature": 0.7, + "timeout": 5, + "user": "langfuse_latency_test_user", + "mock_response": "Mock response", + } + completion_args.update(completion_kwargs) + return asyncio.create_task(litellm.acompletion(**completion_args)) From a08ad0ea70f9276275acb7ed31d43bbaeec0cbcc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 20:50:57 -0700 Subject: [PATCH 3/9] add load tests to ci/cd --- .circleci/config.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ac8fbd48c..bbdee78a7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,6 +149,33 @@ jobs: # Store test results - store_test_results: path: test-results + load_testing: + docker: + - image: cimg/python:3.11 + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/load_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results installing_litellm_on_python: docker: @@ -289,7 +316,7 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests + python -m pytest -s -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests no_output_timeout: 120m # Store test results @@ -607,10 +634,17 @@ workflows: only: - main - /litellm_.*/ + - load_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - publish_to_pypi: requires: - local_testing - build_and_test + - load_testing - proxy_log_to_otel_tests - proxy_pass_through_endpoint_tests filters: From da29b070bbf0122a24d33ebe6e4f3eeffd1fdf17 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 20:53:52 -0700 Subject: [PATCH 4/9] print load test results --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bbdee78a7c..850415b9d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,7 +170,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/load_tests -x --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/load_tests -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results From a1f8fcfeedc440076d517936ece2e418e134f3ad Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 21:06:42 -0700 Subject: [PATCH 5/9] fix load test --- tests/load_tests/test_langsmith_load_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index ba241c7f45..76d6369746 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -20,9 +20,6 @@ def test_langsmith_logging_async(): os.environ["LANGSMITH_PROJECT"] = "pr-b" os.environ["LANGSMITH_BASE_URL"] = "http://0.0.0.0:8090" - litellm.set_verbose = True - verbose_logger.setLevel(logging.DEBUG) - # Make 5 calls with an empty success_callback litellm.success_callback = [] litellm.callbacks = [] From b01a42ef4f8379b6b0901edd3bc50dd003582b4e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 21:16:16 -0700 Subject: [PATCH 6/9] fix langsmith load test --- tests/load_tests/test_langsmith_load_test.py | 121 ++++++++----------- 1 file changed, 51 insertions(+), 70 deletions(-) diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index 76d6369746..b1a9fabe53 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -13,46 +13,56 @@ import pytest def test_langsmith_logging_async(): - # this tests time added to make langsmith logging calls, vs just acompletion calls try: - os.environ["LANGSMITH_API_KEY"] = "lsv2_anything" os.environ["LANGSMITH_PROJECT"] = "pr-b" os.environ["LANGSMITH_BASE_URL"] = "http://0.0.0.0:8090" - # Make 5 calls with an empty success_callback - litellm.success_callback = [] - litellm.callbacks = [] - litellm._async_success_callback = [] - litellm._async_failure_callback = [] - litellm._async_failure_callback = [] - litellm.failure_callback = [] - start_time_empty_callback = asyncio.run(make_async_calls()) - print("done with no callback test") + percentage_diffs = [] - print("starting langsmith test") - # Make 5 calls with success_callback set to "langsmith" - litellm.success_callback = ["langsmith"] - start_time_langsmith = asyncio.run(make_async_calls()) - print("done with langsmith test") + for run in range(3): + print(f"\nRun {run + 1}:") - # Compare the time for both scenarios - print(f"Time taken with success_callback='langsmith': {start_time_langsmith}") - print(f"Time taken with empty success_callback: {start_time_empty_callback}") + # Test with empty success_callback + litellm.success_callback = [] + litellm.callbacks = [] + litellm._async_success_callback = [] + litellm._async_failure_callback = [] + litellm.failure_callback = [] + start_time_empty_callback = asyncio.run(make_async_calls()) + print("Done with no callback test") - # Calculate the percentage difference - percentage_diff = ( - abs(start_time_langsmith - start_time_empty_callback) - / start_time_empty_callback - * 100 - ) + # Test with langsmith callback + print("Starting langsmith test") + litellm.success_callback = ["langsmith"] + start_time_langsmith = asyncio.run(make_async_calls()) + print("Done with langsmith test") - # Assert that the difference is not more than 10% + # Compare times and calculate percentage difference + print(f"Time with success_callback='langsmith': {start_time_langsmith}") + print(f"Time with empty success_callback: {start_time_empty_callback}") + + percentage_diff = ( + abs(start_time_langsmith - start_time_empty_callback) + / start_time_empty_callback + * 100 + ) + percentage_diffs.append(percentage_diff) + print(f"Performance difference: {percentage_diff:.2f}%") + print("percentage_diffs", percentage_diffs) + # Calculate average percentage difference + avg_percentage_diff = sum(percentage_diffs) / len(percentage_diffs) + print(f"\nAverage performance difference: {avg_percentage_diff:.2f}%") + + # Assert that the average difference is not more than 10% assert ( - percentage_diff < 10 - ), f"Performance difference of {percentage_diff:.2f}% exceeds 10% threshold" + avg_percentage_diff < 10 + ), f"Average performance difference of {avg_percentage_diff:.2f}% exceeds 10% threshold" - print(f"Performance difference: {percentage_diff:.2f}%") + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"An exception occurred - {e}") except litellm.Timeout as e: pass @@ -61,52 +71,23 @@ def test_langsmith_logging_async(): async def make_async_calls(metadata=None, **completion_kwargs): - tasks = [] - for _ in range(100): - tasks.append(create_async_task()) + total_tasks = 300 + batch_size = 100 + total_time = 0 - # Measure the start time before running the tasks - start_time = asyncio.get_event_loop().time() + for batch in range(3): + tasks = [create_async_task() for _ in range(batch_size)] - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) + start_time = asyncio.get_event_loop().time() + responses = await asyncio.gather(*tasks) - # Print the responses when tasks return - for idx, response in enumerate(responses): - print(f"Response from Task {idx + 1}: {response}") + for idx, response in enumerate(responses): + print(f"Response from Task {batch * batch_size + idx + 1}: {response}") - await asyncio.sleep(1) + await asyncio.sleep(1) - for _ in range(100): - tasks.append(create_async_task()) - - # Measure the start time before running the tasks - start_time = asyncio.get_event_loop().time() - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - - # Print the responses when tasks return - for idx, response in enumerate(responses): - print(f"Response from Task {idx + 1}: {response}") - - await asyncio.sleep(1) - - for _ in range(100): - tasks.append(create_async_task()) - - # Measure the start time before running the tasks - start_time = asyncio.get_event_loop().time() - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - - # Print the responses when tasks return - for idx, response in enumerate(responses): - print(f"Response from Task {idx + 1}: {response}") - - # Calculate the total time taken - total_time = asyncio.get_event_loop().time() - start_time + batch_time = asyncio.get_event_loop().time() - start_time + total_time += batch_time return total_time From 97ecf86d3db1cb93809c0485e93da37f69382431 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 21:19:03 -0700 Subject: [PATCH 7/9] fix langsmith load tests --- tests/load_tests/test_langsmith_load_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/load_tests/test_langsmith_load_test.py b/tests/load_tests/test_langsmith_load_test.py index b1a9fabe53..cf9fe526b7 100644 --- a/tests/load_tests/test_langsmith_load_test.py +++ b/tests/load_tests/test_langsmith_load_test.py @@ -16,7 +16,9 @@ def test_langsmith_logging_async(): try: os.environ["LANGSMITH_API_KEY"] = "lsv2_anything" os.environ["LANGSMITH_PROJECT"] = "pr-b" - os.environ["LANGSMITH_BASE_URL"] = "http://0.0.0.0:8090" + os.environ["LANGSMITH_BASE_URL"] = ( + "https://exampleopenaiendpoint-production.up.railway.app" + ) percentage_diffs = [] From b80f27dce32c22cae42de2a820e0de2279036dbf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 21:25:27 -0700 Subject: [PATCH 8/9] fix otel tests --- tests/load_tests/test_otel_load_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/load_tests/test_otel_load_test.py b/tests/load_tests/test_otel_load_test.py index 24cabaa093..1a7b83f5a8 100644 --- a/tests/load_tests/test_otel_load_test.py +++ b/tests/load_tests/test_otel_load_test.py @@ -17,7 +17,9 @@ def test_otel_logging_async(): try: os.environ["OTEL_EXPORTER"] = "otlp_http" - os.environ["OTEL_ENDPOINT"] = "http://0.0.0.0:8090/traces" + os.environ["OTEL_ENDPOINT"] = ( + "https://exampleopenaiendpoint-production.up.railway.app/traces" + ) os.environ["OTEL_HEADERS"] = "Authorization=K0BSwd" # Make 5 calls with an empty success_callback From 88706488f913b91b0b5df6d7bd637e3931bf971d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 11 Sep 2024 21:27:31 -0700 Subject: [PATCH 9/9] fix otel load test --- tests/load_tests/test_otel_load_test.py | 49 ++++++++++--------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tests/load_tests/test_otel_load_test.py b/tests/load_tests/test_otel_load_test.py index 1a7b83f5a8..f0f4d959ad 100644 --- a/tests/load_tests/test_otel_load_test.py +++ b/tests/load_tests/test_otel_load_test.py @@ -13,48 +13,37 @@ import pytest def test_otel_logging_async(): - # this tests time added to make otel logging calls, vs just acompletion calls try: - os.environ["OTEL_EXPORTER"] = "otlp_http" os.environ["OTEL_ENDPOINT"] = ( "https://exampleopenaiendpoint-production.up.railway.app/traces" ) os.environ["OTEL_HEADERS"] = "Authorization=K0BSwd" - # Make 5 calls with an empty success_callback - litellm.success_callback = [] - litellm.callbacks = [] - litellm._async_success_callback = [] - litellm._async_failure_callback = [] - litellm._async_failure_callback = [] - litellm.failure_callback = [] - start_time_empty_callback = asyncio.run(make_async_calls()) - print("done with no callback test") + def single_run(): + litellm.callbacks = [] + start_time_empty = asyncio.run(make_async_calls()) + print(f"Time with empty callback: {start_time_empty}") - print("starting otel test") - # Make 5 calls with success_callback set to "otel" - litellm.callbacks = ["otel"] - start_time_otel = asyncio.run(make_async_calls()) - print("done with otel test") + litellm.callbacks = ["otel"] + start_time_otel = asyncio.run(make_async_calls()) + print(f"Time with otel callback: {start_time_otel}") - # Compare the time for both scenarios - print(f"Time taken with success_callback='otel': {start_time_otel}") - print(f"Time taken with empty success_callback: {start_time_empty_callback}") + percent_diff = ( + abs(start_time_otel - start_time_empty) / start_time_empty * 100 + ) + print(f"Run performance difference: {percent_diff:.2f}%") + return percent_diff - # Calculate the percentage difference - percentage_diff = ( - abs(start_time_otel - start_time_empty_callback) - / start_time_empty_callback - * 100 - ) + percent_diffs = [single_run() for _ in range(3)] + avg_percent_diff = sum(percent_diffs) / len(percent_diffs) + + print(f"Percentage differences: {percent_diffs}") + print(f"Average performance difference: {avg_percent_diff:.2f}%") - # Assert that the difference is not more than 10% assert ( - percentage_diff < 10 - ), f"Performance difference of {percentage_diff:.2f}% exceeds 10% threshold" - - print(f"Performance difference: {percentage_diff:.2f}%") + avg_percent_diff < 10 + ), f"Average performance difference of {avg_percent_diff:.2f}% exceeds 10% threshold" except litellm.Timeout as e: pass