From 5cc0036c87f6767bcc7e378cade279c07144e593 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 17 Feb 2026 17:38:13 +0530 Subject: [PATCH] Add openai evals endpoints and routing --- litellm/proxy/common_request_processing.py | 12 + .../proxy/openai_evals_endpoints/endpoints.py | 590 ++++++++++++++++++ litellm/proxy/proxy_server.py | 2 + litellm/proxy/route_llm_request.py | 43 ++ 4 files changed, 647 insertions(+) create mode 100644 litellm/proxy/openai_evals_endpoints/endpoints.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index f0fa5e44b0..3bfd4c2e24 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -526,6 +526,12 @@ class ProxyBaseLLMRequestProcessing: "acancel_interaction", "asend_message", "call_mcp_tool", + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", ], version: Optional[str] = None, user_model: Optional[str] = None, @@ -708,6 +714,12 @@ class ProxyBaseLLMRequestProcessing: "acancel_interaction", "acancel_batch", "afile_delete", + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", ], proxy_logging_obj: ProxyLogging, general_settings: dict, diff --git a/litellm/proxy/openai_evals_endpoints/endpoints.py b/litellm/proxy/openai_evals_endpoints/endpoints.py new file mode 100644 index 0000000000..5b40c30a77 --- /dev/null +++ b/litellm/proxy/openai_evals_endpoints/endpoints.py @@ -0,0 +1,590 @@ +""" +OpenAI Evals API endpoints - /v1/evals +""" + +from typing import Optional + +import orjson +from fastapi import APIRouter, Depends, Request, Response + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.llms.openai_evals import ( + CancelEvalResponse, + DeleteEvalResponse, + Eval, + ListEvalsResponse, +) + +router = APIRouter() + + +@router.post( + "/v1/evals", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=Eval, +) +async def create_eval( + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a new evaluation. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl -X POST "http://localhost:4000/v1/evals" \ + -H "Authorization: Bearer your-key" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Test Eval", + "data_source_config": {"type": "file", "file_id": "file-abc123"}, + "testing_criteria": {"graders": [{"type": "llm_as_judge"}]} + }' + ``` + + Returns: Eval object with id, status, timestamps, etc. + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body + body = await request.body() + data = orjson.loads(body) if body else {} + + # Extract model for routing (header > query > body) + # When using extra_body={"model": "..."}, the OpenAI SDK merges it into the body + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acreate_eval", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1/evals", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=ListEvalsResponse, +) +async def list_evals( + fastapi_response: Response, + request: Request, + limit: Optional[int] = 20, + after: Optional[str] = None, + before: Optional[str] = None, + order: Optional[str] = None, + order_by: Optional[str] = None, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + List evaluations with pagination. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl "http://localhost:4000/v1/evals?limit=10" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: ListEvalsResponse with list of evaluations + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for GET) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Use query params if not in body + if "limit" not in data and limit is not None: + data["limit"] = limit + if "after" not in data and after is not None: + data["after"] = after + if "before" not in data and before is not None: + data["before"] = before + if "order" not in data and order is not None: + data["order"] = order + if "order_by" not in data and order_by is not None: + data["order_by"] = order_by + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="alist_evals", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1/evals/{eval_id}", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=Eval, +) +async def get_eval( + eval_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get a specific evaluation by ID. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl "http://localhost:4000/v1/evals/eval_123" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: Eval object + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for GET) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id from path parameter + data["eval_id"] = eval_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="aget_eval", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1/evals/{eval_id}", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=Eval, +) +async def update_eval( + eval_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Update an evaluation. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl -X POST "http://localhost:4000/v1/evals/eval_123" \ + -H "Authorization: Bearer your-key" \ + -H "Content-Type: application/json" \ + -d '{"name": "Updated Name"}' + ``` + + Returns: Updated Eval object + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id from path parameter + data["eval_id"] = eval_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="aupdate_eval", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.delete( + "/v1/evals/{eval_id}", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=DeleteEvalResponse, +) +async def delete_eval( + eval_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete an evaluation. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl -X DELETE "http://localhost:4000/v1/evals/eval_123" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: DeleteEvalResponse with deletion confirmation + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for DELETE) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id from path parameter + data["eval_id"] = eval_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="adelete_eval", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1/evals/{eval_id}/cancel", + tags=["OpenAI Evals API"], + dependencies=[Depends(user_api_key_auth)], + response_model=CancelEvalResponse, +) +async def cancel_eval( + eval_id: str, + fastapi_response: Response, + request: Request, + custom_llm_provider: Optional[str] = "openai", + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Cancel a running evaluation. + + Model-based routing (for multi-account support): + - Pass model via header: `x-litellm-model: gpt-4-account-1` + - Pass model via query: `?model=gpt-4-account-1` + - Pass model via body: `{"model": "gpt-4-account-1"}` + + Example usage: + ```bash + curl -X POST "http://localhost:4000/v1/evals/eval_123/cancel" \ + -H "Authorization: Bearer your-key" + ``` + + Returns: CancelEvalResponse with cancellation confirmation + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + # Read request body (optional for cancel) + body = await request.body() + data = orjson.loads(body) if body else {} + + # Set eval_id from path parameter + data["eval_id"] = eval_id + + # Extract model for routing (header > query > body) + model = ( + data.get("model") + or request.query_params.get("model") + or request.headers.get("x-litellm-model") + ) + if model: + data["model"] = model + + if "custom_llm_provider" not in data: + data["custom_llm_provider"] = custom_llm_provider + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acancel_eval", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=data.get("model"), + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 318c64e4a0..e332dd1763 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -411,6 +411,7 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import ( from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router +from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) @@ -12426,6 +12427,7 @@ app.include_router(llm_passthrough_router) app.include_router(mcp_management_router) app.include_router(anthropic_router) app.include_router(anthropic_skills_router) +app.include_router(evals_router) app.include_router(claude_code_marketplace_router) app.include_router(google_router) app.include_router(langfuse_router) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index ba3d19ef45..eda042f9fe 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -73,6 +73,13 @@ ROUTE_ENDPOINT_MAPPING = { "aget_interaction": "/interactions/{interaction_id}", "adelete_interaction": "/interactions/{interaction_id}", "acancel_interaction": "/interactions/{interaction_id}/cancel", + # OpenAI Evals API routes + "acreate_eval": "/evals", + "alist_evals": "/evals", + "aget_eval": "/evals/{eval_id}", + "aupdate_eval": "/evals/{eval_id}", + "adelete_eval": "/evals/{eval_id}", + "acancel_eval": "/evals/{eval_id}/cancel", } @@ -190,6 +197,12 @@ async def route_request( "acancel_interaction", "acancel_batch", "afile_delete", + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", ], ): """ @@ -256,6 +269,36 @@ async def route_request( else: return getattr(litellm, f"{route_type}")(**data) elif llm_router is not None: + # Evals API: always route to litellm directly (not through router) + # But extract model credentials if a model is provided + if route_type in [ + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + ]: + # If a model is provided, get its credentials from the router + model = data.get("model") + if model and llm_router: + try: + # Try to get deployment credentials for this model + deployment_creds = llm_router.get_deployment_credentials(model_id=model) + if not deployment_creds: + # Try by model group name + deployment = llm_router.get_deployment_by_model_group_name(model_group_name=model) + if deployment and deployment.litellm_params: + deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) + + # If we found credentials, merge them into data (but don't override user-provided values) + if deployment_creds: + data.update(deployment_creds) + except Exception: + # If we can't get deployment creds, continue without them + pass + + return getattr(litellm, f"{route_type}")(**data) # Skip model-based routing for container operations if route_type in [ "acreate_container",