From a9c5b02303a1eb6f63ca3285af28631214ca9a9f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 08:51:14 -0800 Subject: [PATCH 1/9] (v0) fix --- litellm/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 2e0650be6e..c4144267bf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2357,12 +2357,16 @@ def client(original_function): print_verbose( f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) - asyncio.create_task( - logging_obj.async_success_handler(result, start_time, end_time) - ) + # asyncio.to_thread( + # logging_obj.async_success_handler(result, start_time, end_time) + # ) threading.Thread( target=logging_obj.success_handler, args=(result, start_time, end_time) ).start() + threading.Thread( + target=logging_obj.async_success_handler, + args=(result, start_time, end_time), + ).start() # RETURN RESULT if hasattr(result, "_hidden_params"): result._hidden_params["model_id"] = kwargs.get("model_info", {}).get( From cd08a0276449b617c1f2b81ac3997a51e34099ac Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 08:51:52 -0800 Subject: [PATCH 2/9] (test) add blocking callback test --- litellm/tests/test_async_callbacks.py | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 litellm/tests/test_async_callbacks.py diff --git a/litellm/tests/test_async_callbacks.py b/litellm/tests/test_async_callbacks.py new file mode 100644 index 0000000000..1d586070d8 --- /dev/null +++ b/litellm/tests/test_async_callbacks.py @@ -0,0 +1,49 @@ +import json +import sys +import os +import io, asyncio + +import logging + +logging.basicConfig(level=logging.DEBUG) +sys.path.insert(0, os.path.abspath("../..")) + +from litellm import completion +import litellm + +litellm.num_retries = 3 +import time +import pytest + + +async def custom_callback( + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, + end_time, # start/end time +): + # Your custom code here + print("LITELLM: in custom callback function") + print("kwargs", kwargs) + print("completion_response", completion_response) + print("start_time", start_time) + print("end_time", end_time) + time.sleep(1) + + return + + +def test_time_to_run_10_completions(): + litellm.callbacks = [custom_callback] + start = time.time() + + asyncio.run( + litellm.acompletion( + model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hello"}] + ) + ) + end = time.time() + print(f"Time to run 10 completions: {end - start}") + + +test_time_to_run_10_completions() From e6b5152e639281eaa4b2ed9b146bb7e6a5f7ffe9 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 08:52:17 -0800 Subject: [PATCH 3/9] (chore) update load test --- litellm/proxy/proxy_config.yaml | 5 +++-- litellm/proxy/tests/load_test_completion.py | 10 ++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 8cd2fcec85..417b4c6f10 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -57,12 +57,13 @@ model_list: mode: embedding litellm_settings: fallbacks: [{"openai-gpt-3.5": ["azure-gpt-3.5"]}] + success_callback: ['langfuse'] # cache: True # setting callback class # callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] -general_settings: - master_key: sk-1234 +# general_settings: +# master_key: sk-1234 # database_type: "dynamo_db" # database_args: { # 👈 all args - https://github.com/BerriAI/litellm/blob/befbcbb7ac8f59835ce47415c128decf37aac328/litellm/proxy/_types.py#L190 # "billing_mode": "PAY_PER_REQUEST", diff --git a/litellm/proxy/tests/load_test_completion.py b/litellm/proxy/tests/load_test_completion.py index f2f480c889..b5d711d813 100644 --- a/litellm/proxy/tests/load_test_completion.py +++ b/litellm/proxy/tests/load_test_completion.py @@ -11,12 +11,10 @@ async def litellm_completion(): # Your existing code for litellm_completion goes here try: response = await litellm_client.chat.completions.create( - model="Azure OpenAI GPT-4 Canada-East (External)", - stream=True, + model="azure-gpt-3.5", messages=[{"role": "user", "content": f"This is a test: {uuid.uuid4()}"}], ) - async for chunk in response: - print(chunk) + print(response) return response except Exception as e: @@ -27,9 +25,9 @@ async def litellm_completion(): async def main(): - for i in range(1000000): + for i in range(150): start = time.time() - n = 1000 # Number of concurrent tasks + n = 150 # Number of concurrent tasks tasks = [litellm_completion() for _ in range(n)] chat_completions = await asyncio.gather(*tasks) From f2cfb76920e8fa73299c6949bff84e89c53aff8b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 09:52:51 -0800 Subject: [PATCH 4/9] (fix) use asyncio run_in_executor --- litellm/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index c4144267bf..c9d9d32093 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2363,10 +2363,12 @@ def client(original_function): threading.Thread( target=logging_obj.success_handler, args=(result, start_time, end_time) ).start() - threading.Thread( - target=logging_obj.async_success_handler, - args=(result, start_time, end_time), - ).start() + + loop = asyncio.get_event_loop() + loop.run_in_executor( + executor=None, + func=logging_obj.async_success_handler(result, start_time, end_time), + ) # RETURN RESULT if hasattr(result, "_hidden_params"): result._hidden_params["model_id"] = kwargs.get("model_info", {}).get( From 6a695477ba0e9db68ee5467b57cf8d94b2d99a95 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 10:44:51 -0800 Subject: [PATCH 5/9] (fix) async langfuse logger --- litellm/utils.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index c9d9d32093..dbfbf46ecb 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1864,12 +1864,6 @@ def client(original_function): # we only support async s3 logging for acompletion/aembedding since that's used on proxy litellm._async_success_callback.append(callback) removed_async_items.append(index) - elif callback == "langfuse" and inspect.iscoroutinefunction( - original_function - ): - # use async success callback for langfuse if this is litellm.acompletion(). Streaming logging does not work otherwise - litellm._async_success_callback.append(callback) - removed_async_items.append(index) # Pop the async items from success_callback in reverse order to avoid index issues for index in reversed(removed_async_items): @@ -2357,18 +2351,13 @@ def client(original_function): print_verbose( f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) - # asyncio.to_thread( - # logging_obj.async_success_handler(result, start_time, end_time) - # ) + asyncio.create_task( + logging_obj.async_success_handler(result, start_time, end_time) + ) threading.Thread( target=logging_obj.success_handler, args=(result, start_time, end_time) ).start() - loop = asyncio.get_event_loop() - loop.run_in_executor( - executor=None, - func=logging_obj.async_success_handler(result, start_time, end_time), - ) # RETURN RESULT if hasattr(result, "_hidden_params"): result._hidden_params["model_id"] = kwargs.get("model_info", {}).get( From cb40f58cd3e92587d01e96aa36498bfb962ee9e7 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 11:25:47 -0800 Subject: [PATCH 6/9] (fix) return usage in mock_completion --- litellm/main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index e8022f464f..6ff0a02192 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -343,6 +343,11 @@ def mock_completion( model_response["choices"][0]["message"]["content"] = mock_response model_response["created"] = int(time.time()) model_response["model"] = model + + model_response.usage = Usage( + prompt_tokens=10, completion_tokens=20, total_tokens=30 + ) + return model_response except: @@ -534,10 +539,6 @@ def completion( non_default_params = { k: v for k, v in kwargs.items() if k not in default_params } # model-specific params - pass them straight to the model/provider - if mock_response: - return mock_completion( - model, messages, stream=stream, mock_response=mock_response - ) if timeout is None: timeout = ( kwargs.get("request_timeout", None) or 600 @@ -674,6 +675,10 @@ def completion( optional_params=optional_params, litellm_params=litellm_params, ) + if mock_response: + return mock_completion( + model, messages, stream=stream, mock_response=mock_response + ) if custom_llm_provider == "azure": # azure configs api_type = get_secret("AZURE_API_TYPE") or "azure" From 2f429f37b7f8795b22c9869511a98d2203cefece Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 11:28:10 -0800 Subject: [PATCH 7/9] (test) test latency added with langfuse call --- litellm/tests/test_alangfuse.py | 64 ++++++++++++++++++++------- litellm/tests/test_async_callbacks.py | 49 -------------------- 2 files changed, 48 insertions(+), 65 deletions(-) delete mode 100644 litellm/tests/test_async_callbacks.py diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index 6a10528dfc..e87420433c 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -99,36 +99,68 @@ def pre_langfuse_setup(): return -@pytest.mark.skip(reason="beta test - checking langfuse output") def test_langfuse_logging_async(): + # this tests time added to make langfuse logging calls, vs just acompletion calls try: pre_langfuse_setup() litellm.set_verbose = True + + # Make 5 calls with an empty success_callback + litellm.success_callback = [] + start_time_empty_callback = asyncio.run(make_async_calls()) + print("done with no callback test") + + print("starting langfuse test") + # Make 5 calls with success_callback set to "langfuse" litellm.success_callback = ["langfuse"] + start_time_langfuse = asyncio.run(make_async_calls()) + print("done with langfuse test") - async def _test_langfuse(): - response = await litellm.acompletion( - model="azure/chatgpt-v-2", - messages=[{"role": "user", "content": "This is a test"}], - max_tokens=100, - temperature=0.7, - timeout=5, - user="test_user", - ) - await asyncio.sleep(1) - return response + # Compare the time for both scenarios + print(f"Time taken with success_callback='langfuse': {start_time_langfuse}") + print(f"Time taken with empty success_callback: {start_time_empty_callback}") - response = asyncio.run(_test_langfuse()) - print(f"response: {response}") + # assert the diff is not more than 1 second - this was 5 seconds before the fix + assert abs(start_time_langfuse - start_time_empty_callback) < 1 - # # check langfuse.log to see if there was a failed response - search_logs("langfuse.log") except litellm.Timeout as e: pass except Exception as e: pytest.fail(f"An exception occurred - {e}") +async def make_async_calls(): + tasks = [] + for _ in range(5): + task = asyncio.create_task( + litellm.acompletion( + model="azure/chatgpt-v-2", + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=5, + temperature=0.7, + timeout=5, + user="test_user", + mock_response="It's simple to use and easy to get started", + ) + ) + tasks.append(task) + + # Measure the start time before running the tasks + start_time = asyncio.get_event_loop().time() + + # Wait for all tasks to complete + responses = await asyncio.gather(*tasks) + + # Print the responses when tasks return + for idx, response in enumerate(responses): + print(f"Response from Task {idx + 1}: {response}") + + # Calculate the total time taken + total_time = asyncio.get_event_loop().time() - start_time + + return total_time + + # def test_langfuse_logging_async_text_completion(): # try: # pre_langfuse_setup() diff --git a/litellm/tests/test_async_callbacks.py b/litellm/tests/test_async_callbacks.py deleted file mode 100644 index 1d586070d8..0000000000 --- a/litellm/tests/test_async_callbacks.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -import sys -import os -import io, asyncio - -import logging - -logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) - -from litellm import completion -import litellm - -litellm.num_retries = 3 -import time -import pytest - - -async def custom_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, - end_time, # start/end time -): - # Your custom code here - print("LITELLM: in custom callback function") - print("kwargs", kwargs) - print("completion_response", completion_response) - print("start_time", start_time) - print("end_time", end_time) - time.sleep(1) - - return - - -def test_time_to_run_10_completions(): - litellm.callbacks = [custom_callback] - start = time.time() - - asyncio.run( - litellm.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hello"}] - ) - ) - end = time.time() - print(f"Time to run 10 completions: {end - start}") - - -test_time_to_run_10_completions() From cb99cd18e74d6d9202ef53aab2cc3c72783d182a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 11:38:18 -0800 Subject: [PATCH 8/9] (feat) log cache_hit as langfuse tags --- litellm/integrations/langfuse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 4bf60adc94..bc48a78f72 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -181,6 +181,8 @@ class LangFuseLogger: if supports_tags: for key, value in metadata.items(): tags.append(f"{key}:{value}") + if "cache_hit" in kwargs: + tags.append(f"cache_hit:{kwargs['cache_hit']}") trace_params.update({"tags": tags}) trace = self.Langfuse.trace(**trace_params) From 8cf8da1378fcce3c6ec2f7278709a32ce67aad2f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 19 Jan 2024 11:38:44 -0800 Subject: [PATCH 9/9] (test) langfuse_latency_test_user --- litellm/tests/langfuse.log | 57 --------------------------------- litellm/tests/test_alangfuse.py | 2 +- 2 files changed, 1 insertion(+), 58 deletions(-) diff --git a/litellm/tests/langfuse.log b/litellm/tests/langfuse.log index bc09c5ba2a..e69de29bb2 100644 --- a/litellm/tests/langfuse.log +++ b/litellm/tests/langfuse.log @@ -1,57 +0,0 @@ -Starting new HTTPS connection (1): api.anthropic.com:443 -Starting new HTTPS connection (1): litellm-logging.onrender.com:443 -https://litellm-logging.onrender.com:443 "POST /logging HTTP/1.1" 200 38 -https://api.anthropic.com:443 "POST /v1/complete HTTP/1.1" 200 None -Starting new HTTPS connection (1): litellm-logging.onrender.com:443 -Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'user', 'content': 'this is a streaming test for llama2 + langfuse'}], 'model': 'gpt-3.5-turbo', 'max_tokens': 20, 'stream': True, 'temperature': 0.2}} -connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=600.0 socket_options=None -connect_tcp.complete return_value= -start_tls.started ssl_context= server_hostname='api.openai.com' timeout=600.0 -start_tls.complete return_value= -send_request_headers.started request= -send_request_headers.complete -send_request_body.started request= -send_request_body.complete -receive_response_headers.started request= -https://litellm-logging.onrender.com:443 "POST /logging HTTP/1.1" 200 38 -receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 23 Dec 2023 06:33:00 GMT'), (b'Content-Type', b'text/event-stream'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0613'), (b'openai-organization', b'reliablekeystest'), (b'openai-processing-ms', b'62'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'9000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-limit-tokens_usage_based', b'1000000'), (b'x-ratelimit-remaining-requests', b'8998'), (b'x-ratelimit-remaining-tokens', b'999967'), (b'x-ratelimit-remaining-tokens_usage_based', b'999967'), (b'x-ratelimit-reset-requests', b'6ms'), (b'x-ratelimit-reset-tokens', b'1ms'), (b'x-ratelimit-reset-tokens_usage_based', b'1ms'), (b'x-request-id', b'dd1029a85edecb986fb662945c9f7b4f'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=dnuSnc6BPNJd4lgWKpv3iE2P5zy4r5aCVekXVi7HG7U-1703313180-1-AbeMpAfvmJ6BShULb7tMaErR5ergUrt6ohiXj1e8zoo9AotZ0Jz0alUSUcp8FXyQX2VQ9P6gBUeoSR9aE98OasU=; path=/; expires=Sat, 23-Dec-23 07:03:00 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=dET0GKSNfbtSWNJuXndP8GY8M0ANzDK4Dl7mvIfhmM0-1703313180257-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'839e920e4f47f4b0-BOM'), (b'alt-svc', b'h3=":443"; ma=86400')]) -HTTP Request: POST https://api.openai.com/v1/chat/completions "200 OK" -receive_response_body.started request= -receive_response_body.complete -response_closed.started -response_closed.complete -Starting new HTTPS connection (1): litellm-logging.onrender.com:443 -Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'user', 'content': "What's the weather like in San Francisco, Tokyo, and Paris?"}], 'model': 'gpt-3.5-turbo-1106', 'tool_choice': 'auto', 'tools': [{'type': 'function', 'function': {'name': 'get_current_weather', 'description': 'Get the current weather in a given location', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city and state, e.g. San Francisco, CA'}, 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}}, 'required': ['location']}}}]}} -connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=600.0 socket_options=None -connect_tcp.complete return_value= -start_tls.started ssl_context= server_hostname='api.openai.com' timeout=600.0 -start_tls.complete return_value= -send_request_headers.started request= -send_request_headers.complete -send_request_body.started request= -send_request_body.complete -receive_response_headers.started request= -https://litellm-logging.onrender.com:443 "POST /logging HTTP/1.1" 200 38 -receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 23 Dec 2023 06:33:03 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-1106'), (b'openai-organization', b'reliablekeystest'), (b'openai-processing-ms', b'2145'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'9000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-limit-tokens_usage_based', b'1000000'), (b'x-ratelimit-remaining-requests', b'8998'), (b'x-ratelimit-remaining-tokens', b'999968'), (b'x-ratelimit-remaining-tokens_usage_based', b'999968'), (b'x-ratelimit-reset-requests', b'6ms'), (b'x-ratelimit-reset-tokens', b'1ms'), (b'x-ratelimit-reset-tokens_usage_based', b'1ms'), (b'x-request-id', b'd0fd54d3a7696ee677f3690e9e0d6d04'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=P_4fUmw4vvrbGKTlavf9VWuuzzro87gvhLE0DEGKA84-1703313183-1-ARgz+AQXAzH1uTTK8iyPE3QnT8TovAP61UvYsFD+d5DWM0lFi5U2+eSgPH+Pqt+Y1fNH1FWBUn9DmVceJKvyLcU=; path=/; expires=Sat, 23-Dec-23 07:03:03 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=g.nvBthte.6BJ7KHg5tihyGwupeGfMNMGnw72QUUBQc-1703313183034-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'839e92128b7ff2e2-BOM'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')]) -receive_response_body.started request= -receive_response_body.complete -response_closed.started -response_closed.complete -HTTP Request: POST https://api.openai.com/v1/chat/completions "200 OK" -nction': {'name': 'get_current_weather', 'description': 'Get the current weather in a given location', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city and state, e.g. San Francisco, CA'}, 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}}, 'required': ['location']}}}]}} -connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=600.0 socket_options=None -connect_tcp.complete return_value= -start_tls.started ssl_context= server_hostname='api.openai.com' timeout=600.0 -start_tls.complete return_value= -send_request_headers.started request= -send_request_headers.complete -send_request_body.started request= -send_request_body.complete -receive_response_headers.started request= -https://litellm-logging.onrender.com:443 "POST /logging HTTP/1.1" 200 38 -receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 23 Dec 2023 06:33:03 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-1106'), (b'openai-organization', b'reliablekeystest'), (b'openai-processing-ms', b'2145'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'9000'), (b'x-ratelimit-limit-tokens', b'1000000'), (b'x-ratelimit-limit-tokens_usage_based', b'1000000'), (b'x-ratelimit-remaining-requests', b'8998'), (b'x-ratelimit-remaining-tokens', b'999968'), (b'x-ratelimit-remaining-tokens_usage_based', b'999968'), (b'x-ratelimit-reset-requests', b'6ms'), (b'x-ratelimit-reset-tokens', b'1ms'), (b'x-ratelimit-reset-tokens_usage_based', b'1ms'), (b'x-request-id', b'd0fd54d3a7696ee677f3690e9e0d6d04'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=P_4fUmw4vvrbGKTlavf9VWuuzzro87gvhLE0DEGKA84-1703313183-1-ARgz+AQXAzH1uTTK8iyPE3QnT8TovAP61UvYsFD+d5DWM0lFi5U2+eSgPH+Pqt+Y1fNH1FWBUn9DmVceJKvyLcU=; path=/; expires=Sat, 23-Dec-23 07:03:03 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=g.nvBthte.6BJ7KHg5tihyGwupeGfMNMGnw72QUUBQc-1703313183034-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'839e92128b7ff2e2-BOM'), (b'Content-Encoding', b'gzip'), (b'alt-svc', b'h3=":443"; ma=86400')]) -receive_response_body.started request= -receive_response_body.complete -response_closed.started -response_closed.complete -HTTP Request: POST https://api.openai.com/v1/chat/completions "200 OK" diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index e87420433c..d1cac36ef4 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -139,7 +139,7 @@ async def make_async_calls(): max_tokens=5, temperature=0.7, timeout=5, - user="test_user", + user="langfuse_latency_test_user", mock_response="It's simple to use and easy to get started", ) )