From 370e7777e371079d3f440d66f123f270ec0169f4 Mon Sep 17 00:00:00 2001 From: Samy Chouiti Date: Thu, 18 Jan 2024 09:51:06 +0100 Subject: [PATCH 1/4] Updating PromptLayer callback to support tags + metadata --- litellm/integrations/prompt_layer.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/prompt_layer.py b/litellm/integrations/prompt_layer.py index 4bf2089de2..5b0bb5ee07 100644 --- a/litellm/integrations/prompt_layer.py +++ b/litellm/integrations/prompt_layer.py @@ -2,12 +2,10 @@ # On success, logs events to Promptlayer import dotenv, os import requests -import requests dotenv.load_dotenv() # Loading env variables using dotenv import traceback - class PromptLayerLogger: # Class variables or attributes def __init__(self): @@ -25,6 +23,16 @@ class PromptLayerLogger: for optional_param in kwargs["optional_params"]: new_kwargs[optional_param] = kwargs["optional_params"][optional_param] + # Extract PromptLayer tags from metadata, if such exists + tags = [] + metadata = {} + if "metadata" in kwargs["litellm_params"]: + if "pl_tags" in kwargs["litellm_params"]["metadata"]: + tags = kwargs["litellm_params"]["metadata"]["pl_tags"] + + # Remove "pl_tags" from metadata + metadata = {k:v for k, v in kwargs["litellm_params"]["metadata"].items() if k != "pl_tags"} + print_verbose( f"Prompt Layer Logging - Enters logging function for model kwargs: {new_kwargs}\n, response: {response_obj}" ) @@ -34,7 +42,7 @@ class PromptLayerLogger: json={ "function_name": "openai.ChatCompletion.create", "kwargs": new_kwargs, - "tags": ["hello", "world"], + "tags": tags, "request_response": dict(response_obj), "request_start_time": int(start_time.timestamp()), "request_end_time": int(end_time.timestamp()), @@ -53,14 +61,13 @@ class PromptLayerLogger: raise Exception("Promptlayer did not successfully log the response!") if "request_id" in response_json: - print(kwargs["litellm_params"]["metadata"]) - if kwargs["litellm_params"]["metadata"] is not None: + if metadata: response = requests.post( "https://api.promptlayer.com/rest/track-metadata", json={ "request_id": response_json["request_id"], "api_key": self.key, - "metadata": kwargs["litellm_params"]["metadata"], + "metadata": metadata, }, ) print_verbose( From f8759f1b373b81e2144a2d9fbc314883e7a28bae Mon Sep 17 00:00:00 2001 From: Samy Chouiti Date: Thu, 18 Jan 2024 10:00:31 +0100 Subject: [PATCH 2/4] Adding PromptLayer test for tags --- litellm/tests/test_promptlayer_integration.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_promptlayer_integration.py b/litellm/tests/test_promptlayer_integration.py index 9f0af1af8d..1367b02e59 100644 --- a/litellm/tests/test_promptlayer_integration.py +++ b/litellm/tests/test_promptlayer_integration.py @@ -11,7 +11,6 @@ litellm.success_callback = ["promptlayer"] litellm.set_verbose = True import time - # def test_promptlayer_logging(): # try: # # Redirect stdout @@ -65,8 +64,33 @@ def test_promptlayer_logging_with_metadata(): print(e) -test_promptlayer_logging_with_metadata() +def test_promptlayer_logging_with_metadata_tags(): + try: + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + response = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hi 👋 - i'm ai21"}], + temperature=0.2, + max_tokens=20, + metadata={"model": "ai21", "pl_tags": ["env:dev"]}, + ) + + # Restore stdout + time.sleep(1) + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + print(output) + if "LiteLLM: Prompt Layer Logging: success" not in output: + raise Exception("Required log message not found!") + + except Exception as e: + print(e) + +test_promptlayer_logging_with_metadata() +test_promptlayer_logging_with_metadata_tags() # def test_chat_openai(): # try: From 002993d787af76dac4f869d8741d04af29c07048 Mon Sep 17 00:00:00 2001 From: Samy Chouiti Date: Sun, 21 Jan 2024 22:52:04 +0100 Subject: [PATCH 3/4] PromptLayer: fixed error catching + converting OpenAIs Pydantic output to dicts --- litellm/integrations/prompt_layer.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/prompt_layer.py b/litellm/integrations/prompt_layer.py index 5b0bb5ee07..39a80940b7 100644 --- a/litellm/integrations/prompt_layer.py +++ b/litellm/integrations/prompt_layer.py @@ -2,6 +2,7 @@ # On success, logs events to Promptlayer import dotenv, os import requests +from pydantic import BaseModel dotenv.load_dotenv() # Loading env variables using dotenv import traceback @@ -37,6 +38,10 @@ class PromptLayerLogger: f"Prompt Layer Logging - Enters logging function for model kwargs: {new_kwargs}\n, response: {response_obj}" ) + # python-openai >= 1.0.0 returns Pydantic objects instead of jsons + if isinstance(response_obj, BaseModel): + response_obj = response_obj.model_dump() + request_response = requests.post( "https://api.promptlayer.com/rest/track-request", json={ @@ -53,12 +58,14 @@ class PromptLayerLogger: # "prompt_version":1, }, ) + + response_json = request_response.json() + if not request_response.json().get("success", False): + raise Exception("Promptlayer did not successfully log the response!") + print_verbose( f"Prompt Layer Logging: success - final response object: {request_response.text}" ) - response_json = request_response.json() - if "success" not in request_response.json(): - raise Exception("Promptlayer did not successfully log the response!") if "request_id" in response_json: if metadata: From bc4dab029c8c760173af54f101500df55cdfabbf Mon Sep 17 00:00:00 2001 From: Samy Chouiti Date: Tue, 30 Jan 2024 16:49:46 +0100 Subject: [PATCH 4/4] Adding mock response + Pytests fails --- litellm/tests/test_promptlayer_integration.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/litellm/tests/test_promptlayer_integration.py b/litellm/tests/test_promptlayer_integration.py index 1367b02e59..e869aa5515 100644 --- a/litellm/tests/test_promptlayer_integration.py +++ b/litellm/tests/test_promptlayer_integration.py @@ -7,6 +7,8 @@ sys.path.insert(0, os.path.abspath("../..")) from litellm import completion import litellm +import pytest + litellm.success_callback = ["promptlayer"] litellm.set_verbose = True import time @@ -57,11 +59,11 @@ def test_promptlayer_logging_with_metadata(): sys.stdout = old_stdout output = new_stdout.getvalue().strip() print(output) - if "LiteLLM: Prompt Layer Logging: success" not in output: - raise Exception("Required log message not found!") + + assert "Prompt Layer Logging: success" in output except Exception as e: - print(e) + pytest.fail(f"Error occurred: {e}") def test_promptlayer_logging_with_metadata_tags(): @@ -76,6 +78,7 @@ def test_promptlayer_logging_with_metadata_tags(): temperature=0.2, max_tokens=20, metadata={"model": "ai21", "pl_tags": ["env:dev"]}, + mock_response="this is a mock response" ) # Restore stdout @@ -83,11 +86,11 @@ def test_promptlayer_logging_with_metadata_tags(): sys.stdout = old_stdout output = new_stdout.getvalue().strip() print(output) - if "LiteLLM: Prompt Layer Logging: success" not in output: - raise Exception("Required log message not found!") + + assert "Prompt Layer Logging: success" in output except Exception as e: - print(e) + pytest.fail(f"Error occurred: {e}") test_promptlayer_logging_with_metadata() test_promptlayer_logging_with_metadata_tags()