tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-21 08:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add test to validate chunks with no delta are processed correctly

Browse Source
This commit is contained in:
Andres Barbaro
2024-05-24 10:57:35 -05:00
parent b002328c8d
commit 565a7f1ce2
1 changed files with 238 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/tests/test_streaming.py
+238
View File
@@ -6,6 +6,7 @@ import traceback
import time, pytest
from pydantic import BaseModel
from typing import Tuple
from unittest.mock import patch, MagicMock
sys.path.insert(
0, os.path.abspath("../..")
@@ -235,6 +236,243 @@ def test_completion_azure_stream_special_char():
assert len(response_str) > 0
def test_completion_azure_stream_content_filter_no_delta():
"""
Tests streaming from Azure when the chunks have no delta because they represent the filtered content
"""
try:
chunks = [
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": "",
"role": "assistant"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": "This"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": " is"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": " a"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": " dummy"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": " response"
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "",
"choices": [
{
"finish_reason": None,
"index": 0,
"content_filter_offsets": {
"check_offset": 35159,
"start_offset": 35159,
"end_offset": 36150
},
"content_filter_results": {
"hate": {
"filtered": False,
"severity": "safe"
},
"self_harm": {
"filtered": False,
"severity": "safe"
},
"sexual": {
"filtered": False,
"severity": "safe"
},
"violence": {
"filtered": False,
"severity": "safe"
}
}
}
],
"created": 0,
"model": "",
"object": ""
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {
"content": "."
},
"finish_reason": None,
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "chatcmpl-9SQxdH5hODqkWyJopWlaVOOUnFwlj",
"choices": [
{
"delta": {},
"finish_reason": "stop",
"index": 0
}
],
"created": 1716563849,
"model": "gpt-4o-2024-05-13",
"object": "chat.completion.chunk",
"system_fingerprint": "fp_5f4bad809a"
},
{
"id": "",
"choices": [
{
"finish_reason": None,
"index": 0,
"content_filter_offsets": {
"check_offset": 36150,
"start_offset": 36060,
"end_offset": 37029
},
"content_filter_results": {
"hate": {
"filtered": False,
"severity": "safe"
},
"self_harm": {
"filtered": False,
"severity": "safe"
},
"sexual": {
"filtered": False,
"severity": "safe"
},
"violence": {
"filtered": False,
"severity": "safe"
}
}
}
],
"created": 0,
"model": "",
"object": ""
}
]
stream_iterator = iter(chunks)
litellm.set_verbose = True
logging_obj = litellm.Logging(
model="berri-benchmarking-Llama-2-70b-chat-hf-4",
messages=messages,
stream=True,
litellm_call_id="1234",
function_id="function_id",
call_type="acompletion",
start_time=time.time(),
)
response = litellm.CustomStreamWrapper(
completion_stream=stream_iterator,
model="azure/gpt-4o",
custom_llm_provider="azure",
logging_obj=logging_obj,
)
complete_response = ""
for idx, chunk in enumerate(response):
# print
chunk, finished = streaming_format_tests(idx, chunk)
complete_response += chunk
if finished:
break
assert len(complete_response) > 0
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")
def test_completion_cohere_stream_bad_key():
try:
litellm.cache = None