litellm/tests/test_litellm/integrations/test_prometheus_missing_metrics.py

"""
Unit tests for the new Prometheus metrics that were previously missing from validation.

Tests for:
- litellm_remaining_api_key_requests_for_model
- litellm_remaining_api_key_tokens_for_model
- litellm_callback_logging_failures_metric
"""
from typing import get_args
from litellm.types.integrations.prometheus import (
    DEFINED_PROMETHEUS_METRICS,
    PrometheusMetricLabels,
    UserAPIKeyLabelNames,
)


def test_new_metrics_in_defined_metrics():
    """
    Test that the new metrics are present in DEFINED_PROMETHEUS_METRICS.
    """
    defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS)

    new_metrics = [
        "litellm_remaining_api_key_requests_for_model",
        "litellm_remaining_api_key_tokens_for_model",
        "litellm_callback_logging_failures_metric",
    ]

    for metric in new_metrics:
        assert (
            metric in defined_metrics
        ), f"{metric} should be in DEFINED_PROMETHEUS_METRICS"


def test_new_metrics_have_correct_labels():
    """
    Test that the new metrics have the correct labels defined.
    """
    # Test API Key limits metrics labels
    api_key_metrics = [
        "litellm_remaining_api_key_requests_for_model",
        "litellm_remaining_api_key_tokens_for_model",
    ]

    expected_api_key_labels = [
        UserAPIKeyLabelNames.API_KEY_HASH.value,
        UserAPIKeyLabelNames.API_KEY_ALIAS.value,
        UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
    ]

    for metric in api_key_metrics:
        labels = PrometheusMetricLabels.get_labels(metric)
        for expected_label in expected_api_key_labels:
            assert (
                expected_label in labels
            ), f"{metric} should have label {expected_label}"

    # Test Callback failure metric labels
    callback_metric = "litellm_callback_logging_failures_metric"
    callback_labels = PrometheusMetricLabels.get_labels(callback_metric)

    assert (
        UserAPIKeyLabelNames.CALLBACK_NAME.value in callback_labels
    ), f"{callback_metric} should have label {UserAPIKeyLabelNames.CALLBACK_NAME.value}"


def test_callback_name_label_definition():
    """
    Test that CALLBACK_NAME is defined correctly in UserAPIKeyLabelNames.
    """
    assert UserAPIKeyLabelNames.CALLBACK_NAME.value == "callback_name"


if __name__ == "__main__":
    test_new_metrics_in_defined_metrics()
    test_new_metrics_have_correct_labels()
    test_callback_name_label_definition()