From 525acaf755b03257eff2bfb52022c42ed0bbc310 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Feb 2026 17:29:00 +0530 Subject: [PATCH] Add BaseEvalsAPIConfig for openai evals --- litellm/llms/base_llm/evals/__init__.py | 7 + litellm/llms/base_llm/evals/transformation.py | 329 ++++++++++++++++++ litellm/types/llms/openai_evals.py | 200 +++++++++++ 3 files changed, 536 insertions(+) create mode 100644 litellm/llms/base_llm/evals/__init__.py create mode 100644 litellm/llms/base_llm/evals/transformation.py create mode 100644 litellm/types/llms/openai_evals.py diff --git a/litellm/llms/base_llm/evals/__init__.py b/litellm/llms/base_llm/evals/__init__.py new file mode 100644 index 0000000000..948ed5364e --- /dev/null +++ b/litellm/llms/base_llm/evals/__init__.py @@ -0,0 +1,7 @@ +""" +Base configuration for Evals API +""" + +from .transformation import BaseEvalsAPIConfig + +__all__ = ["BaseEvalsAPIConfig"] diff --git a/litellm/llms/base_llm/evals/transformation.py b/litellm/llms/base_llm/evals/transformation.py new file mode 100644 index 0000000000..e60f22f0ff --- /dev/null +++ b/litellm/llms/base_llm/evals/transformation.py @@ -0,0 +1,329 @@ +""" +Base configuration class for Evals API +""" + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.openai_evals import ( + CancelEvalResponse, + CreateEvalRequest, + DeleteEvalResponse, + Eval, + ListEvalsParams, + ListEvalsResponse, + UpdateEvalRequest, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class BaseEvalsAPIConfig(ABC): + """Base configuration for Evals API providers""" + + def __init__(self): + pass + + @property + @abstractmethod + def custom_llm_provider(self) -> LlmProviders: + pass + + @abstractmethod + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """ + Validate and update headers with provider-specific requirements + + Args: + headers: Base headers dictionary + litellm_params: LiteLLM parameters + + Returns: + Updated headers dictionary + """ + return headers + + @abstractmethod + def get_complete_url( + self, + api_base: Optional[str], + endpoint: str, + eval_id: Optional[str] = None, + ) -> str: + """ + Get the complete URL for the API request + + Args: + api_base: Base API URL + endpoint: API endpoint (e.g., 'evals', 'evals/{id}') + eval_id: Optional eval ID for specific eval operations + + Returns: + Complete URL + """ + if api_base is None: + raise ValueError("api_base is required") + return f"{api_base}/v1/{endpoint}" + + @abstractmethod + def transform_create_eval_request( + self, + create_request: CreateEvalRequest, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """ + Transform create eval request to provider-specific format + + Args: + create_request: Eval creation parameters + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Provider-specific request body + """ + pass + + @abstractmethod + def transform_create_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """ + Transform provider response to Eval object + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + Eval object + """ + pass + + @abstractmethod + def transform_list_evals_request( + self, + list_params: ListEvalsParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform list evals request parameters + + Args: + list_params: List parameters (pagination, filters) + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Tuple of (url, query_params) + """ + pass + + @abstractmethod + def transform_list_evals_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ListEvalsResponse: + """ + Transform provider response to ListEvalsResponse + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + ListEvalsResponse object + """ + pass + + @abstractmethod + def transform_get_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform get eval request + + Args: + eval_id: Eval ID + api_base: Base API URL + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Tuple of (url, headers) + """ + pass + + @abstractmethod + def transform_get_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """ + Transform provider response to Eval object + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + Eval object + """ + pass + + @abstractmethod + def transform_update_eval_request( + self, + eval_id: str, + update_request: UpdateEvalRequest, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """ + Transform update eval request + + Args: + eval_id: Eval ID + update_request: Update parameters + api_base: Base API URL + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Tuple of (url, headers, body) + """ + pass + + @abstractmethod + def transform_update_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """ + Transform provider response to Eval object + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + Eval object + """ + pass + + @abstractmethod + def transform_delete_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform delete eval request + + Args: + eval_id: Eval ID + api_base: Base API URL + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Tuple of (url, headers) + """ + pass + + @abstractmethod + def transform_delete_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> DeleteEvalResponse: + """ + Transform provider response to DeleteEvalResponse + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + DeleteEvalResponse object + """ + pass + + @abstractmethod + def transform_cancel_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """ + Transform cancel eval request + + Args: + eval_id: Eval ID + api_base: Base API URL + litellm_params: LiteLLM parameters + headers: Request headers + + Returns: + Tuple of (url, headers, body) + """ + pass + + @abstractmethod + def transform_cancel_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CancelEvalResponse: + """ + Transform provider response to CancelEvalResponse + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + CancelEvalResponse object + """ + pass + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict, + ) -> Exception: + """Get appropriate error class for the provider.""" + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/types/llms/openai_evals.py b/litellm/types/llms/openai_evals.py new file mode 100644 index 0000000000..70d61d258a --- /dev/null +++ b/litellm/types/llms/openai_evals.py @@ -0,0 +1,200 @@ +""" +Type definitions for OpenAI Evals API +""" + +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field +from typing_extensions import Required, TypedDict + + +# Evals API Request Types +class DataSourceConfigCustom(TypedDict, total=False): + """Data source configuration for custom data sources""" + + type: Required[Literal["custom"]] + """Data source type - custom""" + + item_schema: Required[Dict[str, Any]] + """JSON schema describing the structure of each row in the dataset""" + + include_sample_schema: Optional[bool] + """Whether eval expects sample schema population""" + + +class DataSourceConfigLogs(TypedDict, total=False): + """Data source configuration for logs-based evals""" + + type: Required[Literal["logs"]] + """Data source type - logs""" + + metadata: Optional[Dict[str, Any]] + """Optional metadata for filtering logs""" + + +class DataSourceConfigStoredCompletions(TypedDict, total=False): + """Data source configuration for stored completions (deprecated)""" + + type: Required[Literal["stored_completions"]] + """Data source type - stored_completions (deprecated)""" + + metadata: Optional[Dict[str, Any]] + """Optional metadata for filtering stored completions""" + + +DataSourceConfig = Union[DataSourceConfigCustom, DataSourceConfigLogs, DataSourceConfigStoredCompletions] + + +class LLMAsJudgeGraderConfig(TypedDict, total=False): + """Configuration for LLM as judge grading""" + + type: Required[Literal["llm_as_judge"]] + """Grader type - LLM as judge""" + + model: Optional[str] + """Model to use as judge (e.g., 'gpt-4')""" + + prompt: Optional[str] + """Custom prompt for the judge model""" + + +class GroundTruthGraderConfig(TypedDict, total=False): + """Configuration for ground truth grading""" + + type: Required[Literal["ground_truth"]] + """Grader type - ground truth comparison""" + + metric: Optional[Literal["exact_match", "f1_score", "bleu"]] + """Metric to use for comparison""" + + +class CustomGraderConfig(TypedDict, total=False): + """Configuration for custom grading function""" + + type: Required[Literal["custom"]] + """Grader type - custom""" + + function_id: Required[str] + """ID of the custom grading function""" + + +GraderConfig = Union[LLMAsJudgeGraderConfig, GroundTruthGraderConfig, CustomGraderConfig] + + +class CreateEvalRequest(TypedDict, total=False): + """Request parameters for creating an evaluation""" + + name: Optional[str] + """The name of the evaluation""" + + data_source_config: Required[DataSourceConfig] + """Configuration for the data source""" + + testing_criteria: Required[List[GraderConfig]] + """List of graders for all eval runs""" + + metadata: Optional[Dict[str, Any]] + """Set of 16 key-value pairs that can be attached to an object (max 64 char keys, 512 char values)""" + + +class UpdateEvalRequest(TypedDict, total=False): + """Request parameters for updating an evaluation""" + + name: Optional[str] + """Updated name""" + + metadata: Optional[Dict[str, Any]] + """Updated metadata""" + + +class ListEvalsParams(TypedDict, total=False): + """Query parameters for listing evaluations""" + + limit: Optional[int] + """Number of results to return per page. Maximum value is 100. Defaults to 20.""" + + after: Optional[str] + """Cursor for pagination - returns evals after this ID""" + + before: Optional[str] + """Cursor for pagination - returns evals before this ID""" + + order: Optional[Literal["asc", "desc"]] + """Sort order for results. Defaults to 'desc'.""" + + order_by: Optional[Literal["created_at", "updated_at"]] + """Field to sort by. Defaults to 'created_at'.""" + + +# Evals API Response Types +class Eval(BaseModel): + """Represents an evaluation from the OpenAI Evals API""" + + id: str + """Unique identifier for the evaluation""" + + object: str = "eval" + """Object type, always 'eval'""" + + created_at: int + """Unix timestamp of when the evaluation was created""" + + updated_at: Optional[int] = None + """Unix timestamp of when the evaluation was last updated""" + + name: Optional[str] = None + """The name of the evaluation""" + + data_source_config: Dict[str, Any] + """Configuration for the data source""" + + testing_criteria: List[Dict[str, Any]] + """List of graders for the evaluation""" + + metadata: Optional[Dict[str, Any]] = None + """Additional metadata""" + + +class ListEvalsResponse(BaseModel): + """Response from listing evaluations""" + + object: str = "list" + """Object type, always 'list'""" + + data: List[Eval] + """List of evaluations""" + + first_id: Optional[str] = None + """ID of the first evaluation in the list""" + + last_id: Optional[str] = None + """ID of the last evaluation in the list""" + + has_more: bool = False + """Whether there are more evaluations available""" + + +class DeleteEvalResponse(BaseModel): + """Response from deleting an evaluation""" + + eval_id: str + """The ID of the deleted evaluation""" + + object: str = "eval.deleted" + """Object type, always 'eval.deleted'""" + + deleted: bool + """Whether the evaluation was successfully deleted""" + + +class CancelEvalResponse(BaseModel): + """Response from cancelling an evaluation""" + + id: str + """The ID of the cancelled evaluation""" + + object: str = "eval" + """Object type, always 'eval'""" + + status: Literal["cancelled"] + """Status after cancellation, always 'cancelled'"""