tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-18 06:26:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix otel - follow genai semantic conventions + support 'instructions' param for tts (#10608)

Browse Source 
* fix(opentelemetry.py): fix passing otel semantic conventions

Fixes SpanAttributes.LLM_PROMPTS to SpanAttributes.LLM_PROMPTS.value

* fix(opentelemetry.py): ensure spanattributes always pass the actual enum value

* fix(main.py): support passing 'instructions' param for gpt-4o-mini-tts

* test: update tests
This commit is contained in:
Krish Dholakia
2025-05-06 21:57:01 -07:00
committed by GitHub
parent c67ca115ae
commit 500e6cddf5
5 changed files with 207 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/integrations/opentelemetry.py
+19 -19
View File
@@ -417,7 +417,7 @@ class OpenTelemetry(CustomLogger):
if not function:
continue
prefix = f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}"
prefix = f"{SpanAttributes.LLM_REQUEST_FUNCTIONS.value}.{i}"
self.safe_set_attribute(
span=span,
key=f"{prefix}.name",
@@ -473,7 +473,7 @@ class OpenTelemetry(CustomLogger):
_value = _function.get(key)
if _value:
kv_pairs[
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.function_call.{key}"
f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.function_call.{key}"
] = _value
return kv_pairs
@@ -525,21 +525,21 @@ class OpenTelemetry(CustomLogger):
if kwargs.get("model"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_MODEL,
key=SpanAttributes.LLM_REQUEST_MODEL.value,
value=kwargs.get("model"),
)
# The LLM request type
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_TYPE,
key=SpanAttributes.LLM_REQUEST_TYPE.value,
value=standard_logging_payload["call_type"],
)
# The Generative AI Provider: Azure, OpenAI, etc.
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_SYSTEM,
key=SpanAttributes.LLM_SYSTEM.value,
value=litellm_params.get("custom_llm_provider", "Unknown"),
)
@@ -547,7 +547,7 @@ class OpenTelemetry(CustomLogger):
if optional_params.get("max_tokens"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_MAX_TOKENS,
key=SpanAttributes.LLM_REQUEST_MAX_TOKENS.value,
value=optional_params.get("max_tokens"),
)
@@ -555,7 +555,7 @@ class OpenTelemetry(CustomLogger):
if optional_params.get("temperature"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_TEMPERATURE,
key=SpanAttributes.LLM_REQUEST_TEMPERATURE.value,
value=optional_params.get("temperature"),
)
@@ -563,20 +563,20 @@ class OpenTelemetry(CustomLogger):
if optional_params.get("top_p"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_REQUEST_TOP_P,
key=SpanAttributes.LLM_REQUEST_TOP_P.value,
value=optional_params.get("top_p"),
)
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_IS_STREAMING,
key=SpanAttributes.LLM_IS_STREAMING.value,
value=str(optional_params.get("stream", False)),
)
if optional_params.get("user"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_USER,
key=SpanAttributes.LLM_USER.value,
value=optional_params.get("user"),
)
@@ -590,7 +590,7 @@ class OpenTelemetry(CustomLogger):
if response_obj and response_obj.get("model"):
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_RESPONSE_MODEL,
key=SpanAttributes.LLM_RESPONSE_MODEL.value,
value=response_obj.get("model"),
)
@@ -598,21 +598,21 @@ class OpenTelemetry(CustomLogger):
if usage:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_USAGE_TOTAL_TOKENS,
key=SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value,
value=usage.get("total_tokens"),
)
# The number of tokens used in the LLM response (completion).
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
key=SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value,
value=usage.get("completion_tokens"),
)
# The number of tokens used in the LLM prompt.
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
key=SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value,
value=usage.get("prompt_tokens"),
)
@@ -634,7 +634,7 @@ class OpenTelemetry(CustomLogger):
if prompt.get("role"):
self.safe_set_attribute(
span=span,
key=f"{SpanAttributes.LLM_PROMPTS}.{idx}.role",
key=f"{SpanAttributes.LLM_PROMPTS.value}.{idx}.role",
value=prompt.get("role"),
)
@@ -643,7 +643,7 @@ class OpenTelemetry(CustomLogger):
prompt["content"] = str(prompt.get("content"))
self.safe_set_attribute(
span=span,
key=f"{SpanAttributes.LLM_PROMPTS}.{idx}.content",
key=f"{SpanAttributes.LLM_PROMPTS.value}.{idx}.content",
value=prompt.get("content"),
)
#############################################
@@ -655,14 +655,14 @@ class OpenTelemetry(CustomLogger):
if choice.get("finish_reason"):
self.safe_set_attribute(
span=span,
key=f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason",
key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.finish_reason",
value=choice.get("finish_reason"),
)
if choice.get("message"):
if choice.get("message").get("role"):
self.safe_set_attribute(
span=span,
key=f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role",
key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.role",
value=choice.get("message").get("role"),
)
if choice.get("message").get("content"):
@@ -674,7 +674,7 @@ class OpenTelemetry(CustomLogger):
)
self.safe_set_attribute(
span=span,
key=f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content",
key=f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.content",
value=choice.get("message").get("content"),
)
litellm/main.py
+15 -13
View File
@@ -2722,9 +2722,9 @@ def completion( # type: ignore # noqa: PLR0915
"aws_region_name" not in optional_params
or optional_params["aws_region_name"] is None
):
optional_params["aws_region_name"] = (
aws_bedrock_client.meta.region_name
)
optional_params[
"aws_region_name"
] = aws_bedrock_client.meta.region_name
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
if bedrock_route == "converse":
@@ -4448,9 +4448,9 @@ def adapter_completion(
new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs)
response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs) # type: ignore
translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = (
None
)
translated_response: Optional[
Union[BaseModel, AdapterCompletionStreamWrapper]
] = None
if isinstance(response, ModelResponse):
translated_response = translation_obj.translate_completion_output_params(
response=response
@@ -5372,6 +5372,7 @@ def speech( # noqa: PLR0915
timeout: Optional[Union[float, httpx.Timeout]] = None,
response_format: Optional[str] = None,
speed: Optional[int] = None,
instructions: Optional[str] = None,
client=None,
headers: Optional[dict] = None,
custom_llm_provider: Optional[str] = None,
@@ -5393,7 +5394,8 @@ def speech( # noqa: PLR0915
optional_params["response_format"] = response_format
if speed is not None:
optional_params["speed"] = speed # type: ignore
if instructions is not None:
optional_params["instructions"] = instructions
if timeout is None:
timeout = litellm.request_timeout
@@ -5901,9 +5903,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(content_chunks) > 0:
response["choices"][0]["message"]["content"] = (
processor.get_combined_content(content_chunks)
)
response["choices"][0]["message"][
"content"
] = processor.get_combined_content(content_chunks)
reasoning_chunks = [
chunk
@@ -5914,9 +5916,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(reasoning_chunks) > 0:
response["choices"][0]["message"]["reasoning_content"] = (
processor.get_combined_reasoning_content(reasoning_chunks)
)
response["choices"][0]["message"][
"reasoning_content"
] = processor.get_combined_reasoning_content(reasoning_chunks)
audio_chunks = [
chunk
litellm/proxy/_new_secret_config.yaml
+1
View File
@@ -65,6 +65,7 @@ litellm_settings:
num_retries: 0
check_provider_endpoint: true
cache: true
callbacks: ["otel"]
files_settings:
- custom_llm_provider: gemini
tests/code_coverage_tests/check_spanattributes_value_usage.py
+168
View File
@@ -0,0 +1,168 @@
"""
SpanAttributes Value Usage Checker
This script ensures that all SpanAttributes enum references in the OpenTelemetry integration
are properly accessed with the .value property. This is important because:
1. Without .value, the enum object itself is used instead of its string value
2. This can cause type errors or unexpected behavior in OpenTelemetry exporters
3. It's a consistent pattern that should be followed for all enum usage
Example of correct usage:
span.set_attribute(key=SpanAttributes.LLM_USER.value, value="user123")
Example of incorrect usage:
span.set_attribute(key=SpanAttributes.LLM_USER, value="user123")
The script checks both through AST parsing (for accurate code analysis) and regex
(for backup coverage) to find any violations.
Usage:
python tests/code_coverage_tests/check_spanattributes_value_usage.py
python tests/code_coverage_tests/check_spanattributes_value_usage.py --debug
"""
import argparse
import ast
import os
import re
from typing import List, Tuple
import sys
# Add parent directory to path so we can import litellm
sys.path.insert(0, os.path.abspath("../.."))
import litellm
class SpanAttributesUsageChecker(ast.NodeVisitor):
"""
Checks if SpanAttributes is used without .value when setting attributes in safe_set_attribute calls
and other attribute setting methods in opentelemetry.py.
This is important to ensure consistent enum value access and prevent type errors
when sending data to OpenTelemetry exporters.
"""
def __init__(self, debug=False):
self.violations = []
self.debug = debug
def visit_Call(self, node):
# Check if this is a call to safe_set_attribute or set_attribute
if isinstance(node.func, ast.Attribute) and node.func.attr in ['safe_set_attribute', 'set_attribute']:
# Look for the 'key' parameter
for keyword in node.keywords:
if keyword.arg == 'key':
# Check if the value is a SpanAttributes member without .value
if isinstance(keyword.value, ast.Attribute) and \
isinstance(keyword.value.value, ast.Name) and \
keyword.value.value.id == 'SpanAttributes':
# Get the source code for this attribute
try:
attr_source = ast.unparse(keyword.value)
if not attr_source.endswith('.value'):
if self.debug:
print(f"AST found violation: {node.lineno}: {attr_source}")
self.violations.append((node.lineno, f"{attr_source} used without .value"))
except AttributeError:
# For Python < 3.9, ast.unparse doesn't exist
# Fallback to our best guess
if keyword.value.attr != 'value' and not hasattr(keyword.value, 'value'):
violation_msg = f"SpanAttributes.{keyword.value.attr} used without .value"
if self.debug:
print(f"AST found violation: {node.lineno}: {violation_msg}")
self.violations.append((node.lineno, violation_msg))
# Continue the visit
self.generic_visit(node)
def check_file(file_path: str, debug: bool = False) -> List[Tuple[int, str]]:
"""
Analyze a Python file to check for SpanAttributes usage without .value
Args:
file_path: Path to the Python file to check
debug: Whether to print debug information
Returns:
List of (line_number, message) tuples identifying violations
"""
with open(file_path, 'r') as file:
content = file.read()
# First try AST parsing for accurate code structure analysis
try:
tree = ast.parse(content)
checker = SpanAttributesUsageChecker(debug=debug)
checker.visit(tree)
violations = checker.violations
# Also do a regex check for backup/extra coverage
# This catches cases that might be missed by AST parsing
# Split content into lines for more precise analysis
lines = content.splitlines()
for i, line in enumerate(lines, 1):
# Skip lines that contain ".value" after "SpanAttributes."
# This prevents false positives for correct usage
if re.search(r"SpanAttributes\.[A-Z_][A-Z0-9_]*\.value", line):
if debug:
print(f"Line {i} skipped - contains .value: {line.strip()}")
continue
# Pattern: Looking for "key=SpanAttributes.ENUM_NAME" without .value at the end
pattern = r"key\s*=\s*SpanAttributes\.[A-Z_][A-Z0-9_]*(?!\.value)"
match = re.search(pattern, line)
if match:
# Check if this violation was already found by AST
if not any(i == line_num for line_num, _ in violations):
if debug:
print(f"Regex found violation: {i}: {match.group(0)}")
violations.append((i, f"SpanAttributes used without .value: {match.group(0)}"))
return violations
except SyntaxError:
print(f"Syntax error in {file_path}")
return []
def main():
"""
Main function to run the SpanAttributes usage check on the OpenTelemetry integration file.
Exits with code 1 if violations are found, 0 otherwise.
"""
parser = argparse.ArgumentParser(description='Check for SpanAttributes used without .value')
parser.add_argument('--debug', action='store_true', help='Enable debug output')
args = parser.parse_args()
# Path to the OpenTelemetry integration file
target_file = os.path.join("litellm", "integrations", "opentelemetry.py")
if not os.path.exists(target_file):
# Try alternate path for local development
target_file = os.path.join("..", "..", "litellm", "integrations", "opentelemetry.py")
if not os.path.exists(target_file):
print(f"Error: Could not find file at {target_file}")
exit(1)
violations = check_file(target_file, debug=args.debug)
if violations:
print(f"Found {len(violations)} SpanAttributes without .value in {target_file}:")
# Sort violations by line number for better readability
violations.sort(key=lambda x: x[0])
for line, message in violations:
print(f" Line {line}: {message}")
print("\nDirect enum reference can cause errors. Always use .value with SpanAttributes enums.")
exit(1)
else:
print(f"All SpanAttributes are used correctly with .value in {target_file}")
exit(0)
if __name__ == "__main__":
main()
tests/logging_callback_tests/test_opentelemetry_unit_tests.py
+4 -4
View File
@@ -31,10 +31,10 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest):
kv_pair_dict = OpenTelemetry._tool_calls_kv_pair(tool_calls)
assert kv_pair_dict == {
f"{SpanAttributes.LLM_COMPLETIONS}.0.function_call.arguments": '{"city": "New York"}',
f"{SpanAttributes.LLM_COMPLETIONS}.0.function_call.name": "get_weather",
f"{SpanAttributes.LLM_COMPLETIONS}.1.function_call.arguments": '{"city": "New York"}',
f"{SpanAttributes.LLM_COMPLETIONS}.1.function_call.name": "get_news",
f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.arguments": '{"city": "New York"}',
f"{SpanAttributes.LLM_COMPLETIONS.value}.0.function_call.name": "get_weather",
f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.arguments": '{"city": "New York"}',
f"{SpanAttributes.LLM_COMPLETIONS.value}.1.function_call.name": "get_news",
}
@pytest.mark.asyncio