From b4c5bacea983aa2a424b0d6582a73588da0835c7 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 9 Dec 2023 22:26:52 -0800 Subject: [PATCH] (test) langfuse - read ddebug logs to find status 200 --- litellm/tests/langfuse.log | Bin 0 -> 15600 bytes litellm/tests/test_langfuse.py | 90 ++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 litellm/tests/langfuse.log diff --git a/litellm/tests/langfuse.log b/litellm/tests/langfuse.log new file mode 100644 index 0000000000000000000000000000000000000000..79a18140f260503397014d6b0f77bc83fdd19f53 GIT binary patch literal 15600 zcmeHO>yFzt7Vd991$X`^16Z;o%h!zyq{(a&Y?29Tr&u5e3`#n(HKSWeYC4l3Pq4rG z1bxLmLJz5n?-)%|Y_o;thesB9$iu^P`9y!33(7>8tVjaCkQa-^+d1*lBmqx^X+j=Z z=Tj1e0-`9^qja^xo<^E5NPJ+1m&Q}a>38P$k3qB0pov(sGz-02a4kf}r@h{NGmx{l zm-7Ybl^Q5LdzN9H5!*CLf80Tpc^(5JaEr?NMyDG2=)Bc%H&zdqkVPP1Z^$=k0-Yzm zFAXCozb2kD8BohJ^%0Cgcl-g>UDLC5$MHs{J05xfF8L)CYmzbW!2`}yM$|C1ZXqWu zPZDWD!{{s;4@-R?d?Hfva}GId)*7W~%w)Brr#m(Q`Y_RL2OQn8Y+I+MH_}JWcw&u5 z!5|ooPU!g-m6PjDBb*GC zUM?N_0S2yR3d^|EyH10k^OONqp&=OcfPB(IUz-xGi7RMbqA`eoazGL326!W(vHYf0 z-d_41#+mk6dPEs_OW|b(mm!5_S*KjgG%reX--u|k3UUrId^D1!Ow2I)NR)asTH+2c zbh??Gi-<6WG#9g>NqFjAgIK1SjOU_xXQDa)7>JxD%NrWyFncWH$4eO)OKA-ZcW&^z z#53mjF&Aifu)>yP}gZ+_oVnlD9!0_7SwkCwPL0k>lI7{3_v`KzbJ z_uWCvQ6K2gGruPs*bT6yT)ni~aSLo#xi#sZxt>0%(&4!HB?I(BE|zPcnD{nJEAP)9 z|5$V&?_Eu%D(%Aw!AiZ|VcpcftIgGTOeIwhSVj`6$}`+)c#y6qg{$Goj(a?EHCZZT z3Pw-I9NZ& zIEYy;0)1SHJ(JT*E$4aCC6-B^fro3d`@|ehZD%^L$n)2Wvh#(YE7kdoeGFrV-^SP1 zBk!zioh!Fh?LPtYAgb;Q7J5QoFq+`zFrm*eNwHAesjih(hJAB{wlqtjpC~KZFIk9n zx4iow{WTS6UtfPqOAGX{$n^>n_f)Flb>Bh@UG0dP>-Ed|e82UjubOy}TGMI3D1NT( zcsB5-zI9BsAksREw z1ysMfTk)-KY+gHjnLAYxM-B2wkkH4>)1dNeD&wM>aW<3QEHjf)T|AP$gpOrP=jxIn z28`1s0}C%Fa=W`kd<>Hy-AGFDkEn~03-25ujR2Nlibri!b}usGkt%{gs!C4L4HGCw z&o91U*Bo(= z#CWiW7!Tufd;n3P9G!Q$wOloQoca(+I6!kB^f1A8EC-fhYeY3IQ9lF>7%_bINfQuf za{qFsfR~Bg5QkyDU5XSz4aa{=jzJr#X3Nw*BYeln#q9n2Pa0M)bzjDglv5Zc5eVVV zUWDZL-+uis(MhPqq|oN<^M?;xs^jz<0gb%CN2v54RDEU*x?8hVcgL7bQh~{$SfJXo^S=YQ3~cy1aA|pV0OM~4 zTn4t=g3IdR`k)ZK|^4FYOAz7zc4LChxLfEK!@#B-l%gZ+!ISm}Q%4`KD z?!&JIs%U~=n;?NbWUP?Dy~y~l#0dp@`cjy%1C_QxS&76p;9~;@b`Z$@fTDsj6%w!p z$Kb*)TEJ)C0{%U|+~B)c5(yj6F8=l8-Kzy2UzJxHp3*!BA3gw}AAiu`xhKz6gFK4v zh&mv|12XEO$%sYp*f-`z=kQ_VDh<`KCW5izh!_u{@f2tC98(<`?{y$j5d6+}?O$tw z)&i{sS_`xmXf4oMptZm^Z-M#!goNt&yd^cXq=pkTiI&u`%WqL<)Geu@AZE0rh7B8{ z{;73`E7ADZ;6tf;z=I!(A40%5DO?TTk{SxuMN4XENezc|#+KCZg=EN<)ZlJ@t!PON zEvZ40UN^sEw4{cbG24kew=Anh?;o@Y>2H{jis<^npebXxmaaKb)w`y9qc@>s?TH*RwDvljB5dt-t5vJ` NTc_F!zE?%(`U6dOD#ic+ literal 0 HcmV?d00001 diff --git a/litellm/tests/test_langfuse.py b/litellm/tests/test_langfuse.py index 017d7e1e2e..a5df49624e 100644 --- a/litellm/tests/test_langfuse.py +++ b/litellm/tests/test_langfuse.py @@ -9,33 +9,105 @@ from litellm import completion import litellm litellm.num_retries = 3 litellm.success_callback = ["langfuse"] -# litellm.set_verbose = True +os.environ["LANGFUSE_DEBUG"] = "True" import time import pytest +def search_logs(log_file_path): + """ + Searches the given log file for logs containing the "/api/public" string. + + Parameters: + - log_file_path (str): The path to the log file to be searched. + + Returns: + - None + + Raises: + - Exception: If there are any bad logs found in the log file. + """ + import re + print("\n searching logs") + bad_logs = [] + good_logs = [] + all_logs = [] + try: + with open(log_file_path, 'r') as log_file: + lines = log_file.readlines() + for line in lines: + all_logs.append(line.strip()) + if "/api/public" in line: + print("Found log with /api/public:") + print(line.strip()) + print("\n\n") + match = re.search(r'receive_response_headers.complete return_value=\(b\'HTTP/1.1\', (\d+),', line) + if match: + status_code = int(match.group(1)) + if status_code != 200 and status_code != 201: + print("got a BAD log") + bad_logs.append(line.strip()) + else: + + good_logs.append(line.strip()) + print("\nBad Logs") + print(bad_logs) + if len(bad_logs)>0: + raise Exception(f"bad logs, Bad logs = {bad_logs}") + + print("\nGood Logs") + print(good_logs) + if len(good_logs) <= 0: + raise Exception(f"There were no Good Logs from Langfuse. No logs with /api/public status 200. \nAll logs:{all_logs}") + + except Exception as e: + raise e + +def pre_langfuse_setup(): + """ + Set up the logging for the 'pre_langfuse_setup' function. + """ + # sends logs to langfuse.log + import logging + # Configure the logging to write to a file + logging.basicConfig(filename="langfuse.log", level=logging.DEBUG) + logger = logging.getLogger() + + # Add a FileHandler to the logger + file_handler = logging.FileHandler("langfuse.log", mode='w') + file_handler.setLevel(logging.DEBUG) + logger.addHandler(file_handler) + return + + def test_langfuse_logging_async(): try: + pre_langfuse_setup() litellm.set_verbose = True async def _test_langfuse(): return await litellm.acompletion( model="gpt-3.5-turbo", messages=[{"role": "user", "content":"This is a test"}], - max_tokens=1000, + max_tokens=100, temperature=0.7, timeout=5, ) response = asyncio.run(_test_langfuse()) print(f"response: {response}") + + time.sleep(2) + # check langfuse.log to see if there was a failed response + search_logs("langfuse.log") except litellm.Timeout as e: pass except Exception as e: pytest.fail(f"An exception occurred - {e}") -# test_langfuse_logging_async() +test_langfuse_logging_async() def test_langfuse_logging(): try: - # litellm.set_verbose = True + pre_langfuse_setup() + litellm.set_verbose = True response = completion(model="claude-instant-1.2", messages=[{ "role": "user", @@ -43,19 +115,23 @@ def test_langfuse_logging(): }], max_tokens=10, temperature=0.2, - metadata={"langfuse/key": "foo"} ) print(response) + time.sleep(2) + # check langfuse.log to see if there was a failed response + search_logs("langfuse.log") + except litellm.Timeout as e: pass except Exception as e: - print(e) + pytest.fail(f"An exception occurred - {e}") test_langfuse_logging() def test_langfuse_logging_stream(): try: + pre_langfuse_setup() litellm.set_verbose=True response = completion(model="anyscale/meta-llama/Llama-2-7b-chat-hf", messages=[{ @@ -99,7 +175,7 @@ def test_langfuse_logging_custom_generation_name(): pytest.fail(f"An exception occurred - {e}") print(e) -test_langfuse_logging_custom_generation_name() +# test_langfuse_logging_custom_generation_name() def test_langfuse_logging_function_calling(): function1 = [