diff --git a/build/lib/litellm/__init__.py b/build/lib/litellm/__init__.py index 6ae281afc9..b0c53be1c9 100644 --- a/build/lib/litellm/__init__.py +++ b/build/lib/litellm/__init__.py @@ -36,4 +36,4 @@ open_ai_embedding_models = [ from .timeout import timeout from .utils import client, logging, exception_type # Import all the symbols from main.py from .main import * # Import all the symbols from main.py - +from .integrations import * \ No newline at end of file diff --git a/build/lib/litellm/integrations/__init__.py b/build/lib/litellm/integrations/__init__.py new file mode 100644 index 0000000000..b9742821a6 --- /dev/null +++ b/build/lib/litellm/integrations/__init__.py @@ -0,0 +1 @@ +from . import * \ No newline at end of file diff --git a/build/lib/litellm/integrations/helicone.py b/build/lib/litellm/integrations/helicone.py new file mode 100644 index 0000000000..6b3d619659 --- /dev/null +++ b/build/lib/litellm/integrations/helicone.py @@ -0,0 +1,73 @@ +#### What this does #### +# On success, logs events to Helicone +import dotenv, os +import requests +from anthropic import HUMAN_PROMPT, AI_PROMPT +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +class HeliconeLogger: + # Class variables or attributes + helicone_model_list = ["gpt", "claude"] + def __init__(self): + # Instance variables + self.provider_url = "https://api.openai.com/v1" + self.key = os.getenv('HELICONE_API_KEY') + + def claude_mapping(self, model, messages, response_obj): + prompt = f"{HUMAN_PROMPT}" + for message in messages: + if "role" in message: + if message["role"] == "user": + prompt += f"{HUMAN_PROMPT}{message['content']}" + else: + prompt += f"{AI_PROMPT}{message['content']}" + else: + prompt += f"{HUMAN_PROMPT}{message['content']}" + prompt += f"{AI_PROMPT}" + claude_provider_request = {"model": model, "prompt": prompt} + + claude_response_obj = {"completion": response_obj['choices'][0]['message']['content'], "model": model, "stop_reason": "stop_sequence"} + + return claude_provider_request, claude_response_obj + + def log_success(self, model, messages, response_obj, start_time, end_time, print_verbose): + # Method definition + try: + print_verbose(f"Helicone Logging - Enters logging function for model {model}") + model = model if any(accepted_model in model for accepted_model in self.helicone_model_list) else "gpt-3.5-turbo" + provider_request = {"model": model, "messages": messages} + + if "claude" in model: + provider_request, response_obj = self.claude_mapping(model=model, messages=messages, response_obj=response_obj) + + providerResponse = { + "json": response_obj, + "headers": {"openai-version": "2020-10-01"}, + "status": 200 + } + + # Code to be executed + url = "https://api.hconeai.com/oai/v1/log" + headers = { + 'Authorization': f'Bearer {self.key}', + 'Content-Type': 'application/json' + } + start_time_seconds = int(start_time.timestamp()) + start_time_milliseconds = int((start_time.timestamp() - start_time_seconds) * 1000) + end_time_seconds = int(end_time.timestamp()) + end_time_milliseconds = int((end_time.timestamp() - end_time_seconds) * 1000) + data = { + "providerRequest": {"url": self.provider_url, "json": provider_request, "meta": {"Helicone-Auth": f"Bearer {self.key}"}}, + "providerResponse": providerResponse, + "timing": {"startTime": {"seconds": start_time_seconds, "milliseconds": start_time_milliseconds}, "endTime": {"seconds": end_time_seconds, "milliseconds": end_time_milliseconds}} # {"seconds": .., "milliseconds": ..} + } + response = requests.post(url, headers=headers, json=data) + if response.status_code == 200: + print_verbose("Helicone Logging - Success!") + else: + print_verbose(f"Helicone Logging - Error Request was not successful. Status Code: {response.status_code}") + print_verbose(f"Helicone Logging - Error {response.text}") + except: + # traceback.print_exc() + print_verbose(f"Helicone Logging Error - {traceback.format_exc()}") + pass \ No newline at end of file diff --git a/build/lib/litellm/utils.py b/build/lib/litellm/utils.py index 1cbbb37cfd..b0050226b8 100644 --- a/build/lib/litellm/utils.py +++ b/build/lib/litellm/utils.py @@ -112,6 +112,7 @@ def set_callbacks(callback_list): subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'sentry_sdk']) import sentry_sdk sentry_sdk_instance = sentry_sdk + sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE") if "SENTRY_API_TRACE_RATE" in os.environ else "1.0" sentry_sdk_instance.init(dsn=os.environ.get("SENTRY_API_URL"), traces_sample_rate=float(os.environ.get("SENTRY_API_TRACE_RATE"))) capture_exception = sentry_sdk_instance.capture_exception add_breadcrumb = sentry_sdk_instance.add_breadcrumb @@ -236,14 +237,16 @@ def handle_success(args, kwargs, result, start_time, end_time): print_verbose("reaches helicone for logging!") model = args[0] if len(args) > 0 else kwargs["model"] messages = args[1] if len(args) > 1 else kwargs["messages"] - heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time) + heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) except: + print_verbose(f"Success Callback Error - {traceback.format_exc()}") pass if success_handler and callable(success_handler): success_handler(args, kwargs) pass except: + print_verbose(f"Success Callback Error - {traceback.format_exc()}") pass diff --git a/dist/litellm-0.1.222-py3-none-any.whl b/dist/litellm-0.1.222-py3-none-any.whl deleted file mode 100644 index c16ca32682..0000000000 Binary files a/dist/litellm-0.1.222-py3-none-any.whl and /dev/null differ diff --git a/dist/litellm-0.1.222.tar.gz b/dist/litellm-0.1.222.tar.gz deleted file mode 100644 index 75f64b0b4b..0000000000 Binary files a/dist/litellm-0.1.222.tar.gz and /dev/null differ diff --git a/dist/litellm-0.1.226-py3-none-any.whl b/dist/litellm-0.1.226-py3-none-any.whl new file mode 100644 index 0000000000..8910c20fb6 Binary files /dev/null and b/dist/litellm-0.1.226-py3-none-any.whl differ diff --git a/dist/litellm-0.1.226.tar.gz b/dist/litellm-0.1.226.tar.gz new file mode 100644 index 0000000000..336259f3f3 Binary files /dev/null and b/dist/litellm-0.1.226.tar.gz differ diff --git a/litellm.egg-info/PKG-INFO b/litellm.egg-info/PKG-INFO index bac726f759..901efa24c2 100644 --- a/litellm.egg-info/PKG-INFO +++ b/litellm.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: litellm -Version: 0.1.222 +Version: 0.1.226 Summary: Library to easily interface with LLM API providers Author: BerriAI License-File: LICENSE diff --git a/litellm.egg-info/SOURCES.txt b/litellm.egg-info/SOURCES.txt index 9355b612b0..88f47e84f1 100644 --- a/litellm.egg-info/SOURCES.txt +++ b/litellm.egg-info/SOURCES.txt @@ -10,4 +10,6 @@ litellm.egg-info/PKG-INFO litellm.egg-info/SOURCES.txt litellm.egg-info/dependency_links.txt litellm.egg-info/requires.txt -litellm.egg-info/top_level.txt \ No newline at end of file +litellm.egg-info/top_level.txt +litellm/integrations/__init__.py +litellm/integrations/helicone.py \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index 6ae281afc9..b0c53be1c9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -36,4 +36,4 @@ open_ai_embedding_models = [ from .timeout import timeout from .utils import client, logging, exception_type # Import all the symbols from main.py from .main import * # Import all the symbols from main.py - +from .integrations import * \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index e552de8a29..0000000000 Binary files a/litellm/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc deleted file mode 100644 index fb94555c74..0000000000 Binary files a/litellm/__pycache__/main.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/timeout.cpython-311.pyc b/litellm/__pycache__/timeout.cpython-311.pyc deleted file mode 100644 index 880e08abbb..0000000000 Binary files a/litellm/__pycache__/timeout.cpython-311.pyc and /dev/null differ diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc deleted file mode 100644 index 11fdde95d0..0000000000 Binary files a/litellm/__pycache__/utils.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/__init__.py b/litellm/integrations/__init__.py new file mode 100644 index 0000000000..b9742821a6 --- /dev/null +++ b/litellm/integrations/__init__.py @@ -0,0 +1 @@ +from . import * \ No newline at end of file diff --git a/litellm/integrations/__pycache__/helicone.cpython-311.pyc b/litellm/integrations/__pycache__/helicone.cpython-311.pyc deleted file mode 100644 index 27ab716961..0000000000 Binary files a/litellm/integrations/__pycache__/helicone.cpython-311.pyc and /dev/null differ diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index ec4e9b634d..6b3d619659 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -30,9 +30,10 @@ class HeliconeLogger: return claude_provider_request, claude_response_obj - def log_success(self, model, messages, response_obj, start_time, end_time): + def log_success(self, model, messages, response_obj, start_time, end_time, print_verbose): # Method definition try: + print_verbose(f"Helicone Logging - Enters logging function for model {model}") model = model if any(accepted_model in model for accepted_model in self.helicone_model_list) else "gpt-3.5-turbo" provider_request = {"model": model, "messages": messages} @@ -61,10 +62,12 @@ class HeliconeLogger: "timing": {"startTime": {"seconds": start_time_seconds, "milliseconds": start_time_milliseconds}, "endTime": {"seconds": end_time_seconds, "milliseconds": end_time_milliseconds}} # {"seconds": .., "milliseconds": ..} } response = requests.post(url, headers=headers, json=data) - # if response.status_code == 200: - # print("Success!") - # else: - # print("Request was not successful. Status Code:", response.status_code) + if response.status_code == 200: + print_verbose("Helicone Logging - Success!") + else: + print_verbose(f"Helicone Logging - Error Request was not successful. Status Code: {response.status_code}") + print_verbose(f"Helicone Logging - Error {response.text}") except: # traceback.print_exc() + print_verbose(f"Helicone Logging Error - {traceback.format_exc()}") pass \ No newline at end of file diff --git a/litellm/tests/test_client.py b/litellm/tests/test_client.py index d791f10ebd..66c281ee24 100644 --- a/litellm/tests/test_client.py +++ b/litellm/tests/test_client.py @@ -30,7 +30,7 @@ def test_completion_openai(): except Exception as e: traceback.print_exc() pytest.fail(f"Error occurred: {e}") - +test_completion_openai() def test_completion_claude(): try: @@ -38,14 +38,14 @@ def test_completion_claude(): # Add any assertions here to check the response except Exception as e: pytest.fail(f"Error occurred: {e}") - +test_completion_claude() def test_completion_non_openai(): try: response = completion(model="command-nightly", messages=messages, logger_fn=logger_fn) # Add any assertions here to check the response except Exception as e: pytest.fail(f"Error occurred: {e}") - +test_completion_non_openai() def test_embedding_openai(): try: response = embedding(model='text-embedding-ada-002', input=[user_message], logger_fn=logger_fn) diff --git a/litellm/tests/test_helicone_integration.py b/litellm/tests/test_helicone_integration.py new file mode 100644 index 0000000000..0b1d6ce8a6 --- /dev/null +++ b/litellm/tests/test_helicone_integration.py @@ -0,0 +1,24 @@ +#### What this tests #### +# This tests if logging to the helicone integration actually works + +import sys, os +import traceback +import pytest + +sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +import litellm +from litellm import embedding, completion + +litellm.success_callback = ["helicone"] + +litellm.set_verbose = True + +user_message = "Hello, how are you?" +messages = [{ "content": user_message,"role": "user"}] + + +#openai call +response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) + +#cohere call +response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]) \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 16fab0aeee..b0050226b8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -237,14 +237,16 @@ def handle_success(args, kwargs, result, start_time, end_time): print_verbose("reaches helicone for logging!") model = args[0] if len(args) > 0 else kwargs["model"] messages = args[1] if len(args) > 1 else kwargs["messages"] - heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time) + heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) except: + print_verbose(f"Success Callback Error - {traceback.format_exc()}") pass if success_handler and callable(success_handler): success_handler(args, kwargs) pass except: + print_verbose(f"Success Callback Error - {traceback.format_exc()}") pass diff --git a/setup.py b/setup.py index 641473e0a4..98597b52ca 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,15 @@ from setuptools import setup, find_packages setup( name='litellm', - version='0.1.222', + version='0.1.226', description='Library to easily interface with LLM API providers', author='BerriAI', packages=[ 'litellm' ], + package_data={ + "litellm": ["integrations/*"], # Specify the directory path relative to your package + }, install_requires=[ 'openai', 'cohere',