diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py new file mode 100644 index 0000000000..975bf172f1 --- /dev/null +++ b/litellm/llms/openai/evals/transformation.py @@ -0,0 +1,258 @@ +""" +OpenAI Evals API configuration and transformations +""" + +from typing import Any, Dict, Optional, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.evals.transformation import ( + BaseEvalsAPIConfig, + LiteLLMLoggingObj, +) +from litellm.types.llms.openai_evals import ( + CancelEvalResponse, + CreateEvalRequest, + DeleteEvalResponse, + Eval, + ListEvalsParams, + ListEvalsResponse, + UpdateEvalRequest, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + + +class OpenAIEvalsConfig(BaseEvalsAPIConfig): + """OpenAI-specific Evals API configuration""" + + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.OPENAI + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """Add OpenAI-specific headers""" + import litellm + from litellm.secret_managers.main import get_secret_str + + # Get API key following OpenAI pattern + api_key = None + if litellm_params: + api_key = litellm_params.api_key + + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + + if not api_key: + raise ValueError("OPENAI_API_KEY is required for Evals API") + + # Add required headers + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + endpoint: str, + eval_id: Optional[str] = None, + ) -> str: + """Get complete URL for OpenAI Evals API""" + if api_base is None: + api_base = "https://api.openai.com" + + if eval_id: + return f"{api_base}/v1/evals/{eval_id}" + return f"{api_base}/v1/{endpoint}" + + def transform_create_eval_request( + self, + create_request: CreateEvalRequest, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """Transform create eval request for OpenAI""" + verbose_logger.debug("Transforming create eval request: %s", create_request) + + # OpenAI expects the request body directly + request_body = {k: v for k, v in create_request.items() if v is not None} + + return request_body + + def transform_create_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """Transform OpenAI response to Eval object""" + response_json = raw_response.json() + verbose_logger.debug("Transforming create eval response: %s", response_json) + + return Eval(**response_json) + + def transform_list_evals_request( + self, + list_params: ListEvalsParams, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform list evals request for OpenAI""" + api_base = "https://api.openai.com" + if litellm_params and litellm_params.api_base: + api_base = litellm_params.api_base + + url = self.get_complete_url(api_base=api_base, endpoint="evals") + + # Build query parameters + query_params: Dict[str, Any] = {} + if "limit" in list_params and list_params["limit"]: + query_params["limit"] = list_params["limit"] + if "after" in list_params and list_params["after"]: + query_params["after"] = list_params["after"] + if "before" in list_params and list_params["before"]: + query_params["before"] = list_params["before"] + if "order" in list_params and list_params["order"]: + query_params["order"] = list_params["order"] + if "order_by" in list_params and list_params["order_by"]: + query_params["order_by"] = list_params["order_by"] + + verbose_logger.debug( + "List evals request made to OpenAI Evals endpoint with params: %s", + query_params, + ) + + return url, query_params + + def transform_list_evals_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ListEvalsResponse: + """Transform OpenAI response to ListEvalsResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming list evals response: %s", response_json) + + return ListEvalsResponse(**response_json) + + def transform_get_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform get eval request for OpenAI""" + url = self.get_complete_url( + api_base=api_base, endpoint="evals", eval_id=eval_id + ) + + verbose_logger.debug("Get eval request - URL: %s", url) + + return url, headers + + def transform_get_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """Transform OpenAI response to Eval object""" + response_json = raw_response.json() + verbose_logger.debug("Transforming get eval response: %s", response_json) + + return Eval(**response_json) + + def transform_update_eval_request( + self, + eval_id: str, + update_request: UpdateEvalRequest, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """Transform update eval request for OpenAI""" + url = self.get_complete_url( + api_base=api_base, endpoint="evals", eval_id=eval_id + ) + + # Build request body + request_body = {k: v for k, v in update_request.items() if v is not None} + + verbose_logger.debug( + "Update eval request - URL: %s, body: %s", url, request_body + ) + + return url, headers, request_body + + def transform_update_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> Eval: + """Transform OpenAI response to Eval object""" + response_json = raw_response.json() + verbose_logger.debug("Transforming update eval response: %s", response_json) + + return Eval(**response_json) + + def transform_delete_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform delete eval request for OpenAI""" + url = self.get_complete_url( + api_base=api_base, endpoint="evals", eval_id=eval_id + ) + + verbose_logger.debug("Delete eval request - URL: %s", url) + + return url, headers + + def transform_delete_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> DeleteEvalResponse: + """Transform OpenAI response to DeleteEvalResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming delete eval response: %s", response_json) + + return DeleteEvalResponse(**response_json) + + def transform_cancel_eval_request( + self, + eval_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict, Dict]: + """Transform cancel eval request for OpenAI""" + url = f"{self.get_complete_url(api_base=api_base, endpoint='evals', eval_id=eval_id)}/cancel" + + # Empty body for cancel request + request_body: Dict[str, Any] = {} + + verbose_logger.debug("Cancel eval request - URL: %s", url) + + return url, headers, request_body + + def transform_cancel_eval_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CancelEvalResponse: + """Transform OpenAI response to CancelEvalResponse""" + response_json = raw_response.json() + verbose_logger.debug("Transforming cancel eval response: %s", response_json) + + return CancelEvalResponse(**response_json) diff --git a/litellm/proxy/openai_evals_endpoints/__init__.py b/litellm/proxy/openai_evals_endpoints/__init__.py new file mode 100644 index 0000000000..aff422e67a --- /dev/null +++ b/litellm/proxy/openai_evals_endpoints/__init__.py @@ -0,0 +1,7 @@ +""" +OpenAI Evals API endpoints +""" + +from .endpoints import router + +__all__ = ["router"] diff --git a/litellm/utils.py b/litellm/utils.py index 9b82f5d321..2a29c08904 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -380,6 +380,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig +from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig @@ -8282,6 +8283,25 @@ class ProviderConfigManager: return litellm.AnthropicSkillsConfig() return None + @staticmethod + def get_provider_evals_api_config( + provider: LlmProviders, + ) -> Optional["BaseEvalsAPIConfig"]: + """ + Get provider-specific Evals API configuration + + Args: + provider: The LLM provider + + Returns: + Provider-specific Evals API config or None + """ + if litellm.LlmProviders.OPENAI == provider: + from litellm.llms.openai.evals.transformation import OpenAIEvalsConfig + + return OpenAIEvalsConfig() + return None + @staticmethod def get_provider_text_completion_config( model: str,