tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-12 03:06:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

test_proxy_failure_metrics

Browse Source
This commit is contained in:
Ishaan Jaffer
2026-01-31 18:10:17 -08:00
parent 35e29c2bcd
commit faff9d1dc5
1 changed files with 34 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/otel_tests/test_prometheus.py
+34 -16
View File
@@ -106,23 +106,41 @@ async def test_proxy_failure_metrics():
print("/metrics", metrics)
# Check if the failure metric is present and correct - use pattern matching for robustness
# Labels are ordered alphabetically by Prometheus: api_key_alias, client_ip, end_user, exception_class,
# exception_status, hashed_api_key, model_id, requested_model, route, team, team_alias, user, user_agent, user_email
expected_metric_pattern = 'litellm_proxy_failed_requests_metric_total{api_key_alias="None",client_ip="None",end_user="None",exception_class="Openai.RateLimitError",exception_status="429",hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",model_id="None",requested_model="fake-azure-endpoint",route="/chat/completions",team="None",team_alias="None",user="default_user_id",user_agent="None",user_email="None"}'
# Labels are ordered alphabetically by Prometheus: api_key_alias, end_user, exception_class,
# exception_status, hashed_api_key, requested_model, route, team, team_alias, user, user_email
# Note: client_ip, user_agent, model_id are present but we use substring matching to be flexible
expected_metric_pattern = 'litellm_proxy_failed_requests_metric_total{api_key_alias="None"'
# Check if the pattern is in metrics and contains required fields
found_metric = False
for line in metrics.split("\n"):
if expected_metric_pattern in line and \
'exception_class="Openai.RateLimitError"' in line and \
'exception_status="429"' in line and \
'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"' in line and \
'requested_model="fake-azure-endpoint"' in line and \
'route="/chat/completions"' in line and \
'user_email="None"' in line:
found_metric = True
break
assert found_metric, f"Expected failure metric not found in /metrics. Looking for: {expected_metric_pattern} with required fields"
# Check if the pattern is in metrics
assert any(
expected_metric_pattern in line for line in metrics.split("\n")
), f"Expected failure metric pattern not found in /metrics. Pattern: {expected_metric_pattern}"
# Check total requests metric
# Labels are ordered alphabetically: api_key_alias, client_ip, end_user, hashed_api_key, model_id,
# requested_model, route, status_code, team, team_alias, user, user_agent, user_email
total_requests_pattern = 'litellm_proxy_total_requests_metric_total{api_key_alias="None",client_ip="None",end_user="None",hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",model_id="None",requested_model="fake-azure-endpoint",route="/chat/completions",status_code="429",team="None",team_alias="None",user="default_user_id",user_agent="None",user_email="None"}'
assert any(
total_requests_pattern in line for line in metrics.split("\n")
), f"Expected total requests metric pattern not found in /metrics. Pattern: {total_requests_pattern}"
# Check total requests metric similarly
total_requests_pattern = 'litellm_proxy_total_requests_metric_total{api_key_alias="None"'
found_total_metric = False
for line in metrics.split("\n"):
if total_requests_pattern in line and \
'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"' in line and \
'requested_model="fake-azure-endpoint"' in line and \
'route="/chat/completions"' in line and \
'status_code="429"' in line and \
'user_email="None"' in line:
found_total_metric = True
break
assert found_total_metric, f"Expected total requests metric not found in /metrics. Looking for: {total_requests_pattern} with required fields"
@pytest.mark.asyncio