Add BaseEvalsAPIConfig for openai evals

litellm/llms/base_llm/evals/__init__.py

@@ -0,0 +1,7 @@
+"""
+Base configuration for Evals API
+"""
+
+from .transformation import BaseEvalsAPIConfig
+
+__all__ = ["BaseEvalsAPIConfig"]

litellm/llms/base_llm/evals/transformation.py

@@ -0,0 +1,329 @@
+"""
+Base configuration class for Evals API
+"""
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai_evals import (
+    CancelEvalResponse,
+    CreateEvalRequest,
+    DeleteEvalResponse,
+    Eval,
+    ListEvalsParams,
+    ListEvalsResponse,
+    UpdateEvalRequest,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = Any
+
+
+class BaseEvalsAPIConfig(ABC):
+    """Base configuration for Evals API providers"""
+
+    def __init__(self):
+        pass
+
+    @property
+    @abstractmethod
+    def custom_llm_provider(self) -> LlmProviders:
+        pass
+
+    @abstractmethod
+    def validate_environment(
+        self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
+    ) -> dict:
+        """
+        Validate and update headers with provider-specific requirements
+
+        Args:
+            headers: Base headers dictionary
+            litellm_params: LiteLLM parameters
+
+        Returns:
+            Updated headers dictionary
+        """
+        return headers
+
+    @abstractmethod
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        endpoint: str,
+        eval_id: Optional[str] = None,
+    ) -> str:
+        """
+        Get the complete URL for the API request
+
+        Args:
+            api_base: Base API URL
+            endpoint: API endpoint (e.g., 'evals', 'evals/{id}')
+            eval_id: Optional eval ID for specific eval operations
+
+        Returns:
+            Complete URL
+        """
+        if api_base is None:
+            raise ValueError("api_base is required")
+        return f"{api_base}/v1/{endpoint}"
+
+    @abstractmethod
+    def transform_create_eval_request(
+        self,
+        create_request: CreateEvalRequest,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        """
+        Transform create eval request to provider-specific format
+
+        Args:
+            create_request: Eval creation parameters
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Provider-specific request body
+        """
+        pass
+
+    @abstractmethod
+    def transform_create_eval_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> Eval:
+        """
+        Transform provider response to Eval object
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            Eval object
+        """
+        pass
+
+    @abstractmethod
+    def transform_list_evals_request(
+        self,
+        list_params: ListEvalsParams,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """
+        Transform list evals request parameters
+
+        Args:
+            list_params: List parameters (pagination, filters)
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Tuple of (url, query_params)
+        """
+        pass
+
+    @abstractmethod
+    def transform_list_evals_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> ListEvalsResponse:
+        """
+        Transform provider response to ListEvalsResponse
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            ListEvalsResponse object
+        """
+        pass
+
+    @abstractmethod
+    def transform_get_eval_request(
+        self,
+        eval_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """
+        Transform get eval request
+
+        Args:
+            eval_id: Eval ID
+            api_base: Base API URL
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Tuple of (url, headers)
+        """
+        pass
+
+    @abstractmethod
+    def transform_get_eval_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> Eval:
+        """
+        Transform provider response to Eval object
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            Eval object
+        """
+        pass
+
+    @abstractmethod
+    def transform_update_eval_request(
+        self,
+        eval_id: str,
+        update_request: UpdateEvalRequest,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict, Dict]:
+        """
+        Transform update eval request
+
+        Args:
+            eval_id: Eval ID
+            update_request: Update parameters
+            api_base: Base API URL
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Tuple of (url, headers, body)
+        """
+        pass
+
+    @abstractmethod
+    def transform_update_eval_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> Eval:
+        """
+        Transform provider response to Eval object
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            Eval object
+        """
+        pass
+
+    @abstractmethod
+    def transform_delete_eval_request(
+        self,
+        eval_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """
+        Transform delete eval request
+
+        Args:
+            eval_id: Eval ID
+            api_base: Base API URL
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Tuple of (url, headers)
+        """
+        pass
+
+    @abstractmethod
+    def transform_delete_eval_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> DeleteEvalResponse:
+        """
+        Transform provider response to DeleteEvalResponse
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            DeleteEvalResponse object
+        """
+        pass
+
+    @abstractmethod
+    def transform_cancel_eval_request(
+        self,
+        eval_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict, Dict]:
+        """
+        Transform cancel eval request
+
+        Args:
+            eval_id: Eval ID
+            api_base: Base API URL
+            litellm_params: LiteLLM parameters
+            headers: Request headers
+
+        Returns:
+            Tuple of (url, headers, body)
+        """
+        pass
+
+    @abstractmethod
+    def transform_cancel_eval_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> CancelEvalResponse:
+        """
+        Transform provider response to CancelEvalResponse
+
+        Args:
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+
+        Returns:
+            CancelEvalResponse object
+        """
+        pass
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict,
+    ) -> Exception:
+        """Get appropriate error class for the provider."""
+        return BaseLLMException(
+            status_code=status_code,
+            message=error_message,
+            headers=headers,
+        )

litellm/types/llms/openai_evals.py

@@ -0,0 +1,200 @@
+"""
+Type definitions for OpenAI Evals API
+"""
+
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+from typing_extensions import Required, TypedDict
+
+
+# Evals API Request Types
+class DataSourceConfigCustom(TypedDict, total=False):
+    """Data source configuration for custom data sources"""
+
+    type: Required[Literal["custom"]]
+    """Data source type - custom"""
+
+    item_schema: Required[Dict[str, Any]]
+    """JSON schema describing the structure of each row in the dataset"""
+
+    include_sample_schema: Optional[bool]
+    """Whether eval expects sample schema population"""
+
+
+class DataSourceConfigLogs(TypedDict, total=False):
+    """Data source configuration for logs-based evals"""
+
+    type: Required[Literal["logs"]]
+    """Data source type - logs"""
+
+    metadata: Optional[Dict[str, Any]]
+    """Optional metadata for filtering logs"""
+
+
+class DataSourceConfigStoredCompletions(TypedDict, total=False):
+    """Data source configuration for stored completions (deprecated)"""
+
+    type: Required[Literal["stored_completions"]]
+    """Data source type - stored_completions (deprecated)"""
+
+    metadata: Optional[Dict[str, Any]]
+    """Optional metadata for filtering stored completions"""
+
+
+DataSourceConfig = Union[DataSourceConfigCustom, DataSourceConfigLogs, DataSourceConfigStoredCompletions]
+
+
+class LLMAsJudgeGraderConfig(TypedDict, total=False):
+    """Configuration for LLM as judge grading"""
+
+    type: Required[Literal["llm_as_judge"]]
+    """Grader type - LLM as judge"""
+
+    model: Optional[str]
+    """Model to use as judge (e.g., 'gpt-4')"""
+
+    prompt: Optional[str]
+    """Custom prompt for the judge model"""
+
+
+class GroundTruthGraderConfig(TypedDict, total=False):
+    """Configuration for ground truth grading"""
+
+    type: Required[Literal["ground_truth"]]
+    """Grader type - ground truth comparison"""
+
+    metric: Optional[Literal["exact_match", "f1_score", "bleu"]]
+    """Metric to use for comparison"""
+
+
+class CustomGraderConfig(TypedDict, total=False):
+    """Configuration for custom grading function"""
+
+    type: Required[Literal["custom"]]
+    """Grader type - custom"""
+
+    function_id: Required[str]
+    """ID of the custom grading function"""
+
+
+GraderConfig = Union[LLMAsJudgeGraderConfig, GroundTruthGraderConfig, CustomGraderConfig]
+
+
+class CreateEvalRequest(TypedDict, total=False):
+    """Request parameters for creating an evaluation"""
+
+    name: Optional[str]
+    """The name of the evaluation"""
+
+    data_source_config: Required[DataSourceConfig]
+    """Configuration for the data source"""
+
+    testing_criteria: Required[List[GraderConfig]]
+    """List of graders for all eval runs"""
+
+    metadata: Optional[Dict[str, Any]]
+    """Set of 16 key-value pairs that can be attached to an object (max 64 char keys, 512 char values)"""
+
+
+class UpdateEvalRequest(TypedDict, total=False):
+    """Request parameters for updating an evaluation"""
+
+    name: Optional[str]
+    """Updated name"""
+
+    metadata: Optional[Dict[str, Any]]
+    """Updated metadata"""
+
+
+class ListEvalsParams(TypedDict, total=False):
+    """Query parameters for listing evaluations"""
+
+    limit: Optional[int]
+    """Number of results to return per page. Maximum value is 100. Defaults to 20."""
+
+    after: Optional[str]
+    """Cursor for pagination - returns evals after this ID"""
+
+    before: Optional[str]
+    """Cursor for pagination - returns evals before this ID"""
+
+    order: Optional[Literal["asc", "desc"]]
+    """Sort order for results. Defaults to 'desc'."""
+
+    order_by: Optional[Literal["created_at", "updated_at"]]
+    """Field to sort by. Defaults to 'created_at'."""
+
+
+# Evals API Response Types
+class Eval(BaseModel):
+    """Represents an evaluation from the OpenAI Evals API"""
+
+    id: str
+    """Unique identifier for the evaluation"""
+
+    object: str = "eval"
+    """Object type, always 'eval'"""
+
+    created_at: int
+    """Unix timestamp of when the evaluation was created"""
+
+    updated_at: Optional[int] = None
+    """Unix timestamp of when the evaluation was last updated"""
+
+    name: Optional[str] = None
+    """The name of the evaluation"""
+
+    data_source_config: Dict[str, Any]
+    """Configuration for the data source"""
+
+    testing_criteria: List[Dict[str, Any]]
+    """List of graders for the evaluation"""
+
+    metadata: Optional[Dict[str, Any]] = None
+    """Additional metadata"""
+
+
+class ListEvalsResponse(BaseModel):
+    """Response from listing evaluations"""
+
+    object: str = "list"
+    """Object type, always 'list'"""
+
+    data: List[Eval]
+    """List of evaluations"""
+
+    first_id: Optional[str] = None
+    """ID of the first evaluation in the list"""
+
+    last_id: Optional[str] = None
+    """ID of the last evaluation in the list"""
+
+    has_more: bool = False
+    """Whether there are more evaluations available"""
+
+
+class DeleteEvalResponse(BaseModel):
+    """Response from deleting an evaluation"""
+
+    eval_id: str
+    """The ID of the deleted evaluation"""
+
+    object: str = "eval.deleted"
+    """Object type, always 'eval.deleted'"""
+
+    deleted: bool
+    """Whether the evaluation was successfully deleted"""
+
+
+class CancelEvalResponse(BaseModel):
+    """Response from cancelling an evaluation"""
+
+    id: str
+    """The ID of the cancelled evaluation"""
+
+    object: str = "eval"
+    """Object type, always 'eval'"""
+
+    status: Literal["cancelled"]
+    """Status after cancellation, always 'cancelled'"""