mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-10 22:24:51 +00:00
Add openai evals endpoints and routing
This commit is contained in:
Sameer Kankute
@@ -526,6 +526,12 @@ class ProxyBaseLLMRequestProcessing:
|
||||
"acancel_interaction",
|
||||
"asend_message",
|
||||
"call_mcp_tool",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
],
|
||||
version: Optional[str] = None,
|
||||
user_model: Optional[str] = None,
|
||||
@@ -708,6 +714,12 @@ class ProxyBaseLLMRequestProcessing:
|
||||
"acancel_interaction",
|
||||
"acancel_batch",
|
||||
"afile_delete",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
],
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
general_settings: dict,
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
"""
|
||||
OpenAI Evals API endpoints - /v1/evals
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import orjson
|
||||
from fastapi import APIRouter, Depends, Request, Response
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsResponse,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/evals",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=Eval,
|
||||
)
|
||||
async def create_eval(
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Create a new evaluation.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/evals" \
|
||||
-H "Authorization: Bearer your-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"name": "Test Eval",
|
||||
"data_source_config": {"type": "file", "file_id": "file-abc123"},
|
||||
"testing_criteria": {"graders": [{"type": "llm_as_judge"}]}
|
||||
}'
|
||||
```
|
||||
|
||||
Returns: Eval object with id, status, timestamps, etc.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
# When using extra_body={"model": "..."}, the OpenAI SDK merges it into the body
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="acreate_eval",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/v1/evals",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=ListEvalsResponse,
|
||||
)
|
||||
async def list_evals(
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
limit: Optional[int] = 20,
|
||||
after: Optional[str] = None,
|
||||
before: Optional[str] = None,
|
||||
order: Optional[str] = None,
|
||||
order_by: Optional[str] = None,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
List evaluations with pagination.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl "http://localhost:4000/v1/evals?limit=10" \
|
||||
-H "Authorization: Bearer your-key"
|
||||
```
|
||||
|
||||
Returns: ListEvalsResponse with list of evaluations
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body (optional for GET)
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Use query params if not in body
|
||||
if "limit" not in data and limit is not None:
|
||||
data["limit"] = limit
|
||||
if "after" not in data and after is not None:
|
||||
data["after"] = after
|
||||
if "before" not in data and before is not None:
|
||||
data["before"] = before
|
||||
if "order" not in data and order is not None:
|
||||
data["order"] = order
|
||||
if "order_by" not in data and order_by is not None:
|
||||
data["order_by"] = order_by
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="alist_evals",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/v1/evals/{eval_id}",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=Eval,
|
||||
)
|
||||
async def get_eval(
|
||||
eval_id: str,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Get a specific evaluation by ID.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl "http://localhost:4000/v1/evals/eval_123" \
|
||||
-H "Authorization: Bearer your-key"
|
||||
```
|
||||
|
||||
Returns: Eval object
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body (optional for GET)
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Set eval_id from path parameter
|
||||
data["eval_id"] = eval_id
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="aget_eval",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/evals/{eval_id}",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=Eval,
|
||||
)
|
||||
async def update_eval(
|
||||
eval_id: str,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Update an evaluation.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/evals/eval_123" \
|
||||
-H "Authorization: Bearer your-key" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Updated Name"}'
|
||||
```
|
||||
|
||||
Returns: Updated Eval object
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Set eval_id from path parameter
|
||||
data["eval_id"] = eval_id
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="aupdate_eval",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/v1/evals/{eval_id}",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=DeleteEvalResponse,
|
||||
)
|
||||
async def delete_eval(
|
||||
eval_id: str,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Delete an evaluation.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl -X DELETE "http://localhost:4000/v1/evals/eval_123" \
|
||||
-H "Authorization: Bearer your-key"
|
||||
```
|
||||
|
||||
Returns: DeleteEvalResponse with deletion confirmation
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body (optional for DELETE)
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Set eval_id from path parameter
|
||||
data["eval_id"] = eval_id
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="adelete_eval",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/evals/{eval_id}/cancel",
|
||||
tags=["OpenAI Evals API"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=CancelEvalResponse,
|
||||
)
|
||||
async def cancel_eval(
|
||||
eval_id: str,
|
||||
fastapi_response: Response,
|
||||
request: Request,
|
||||
custom_llm_provider: Optional[str] = "openai",
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Cancel a running evaluation.
|
||||
|
||||
Model-based routing (for multi-account support):
|
||||
- Pass model via header: `x-litellm-model: gpt-4-account-1`
|
||||
- Pass model via query: `?model=gpt-4-account-1`
|
||||
- Pass model via body: `{"model": "gpt-4-account-1"}`
|
||||
|
||||
Example usage:
|
||||
```bash
|
||||
curl -X POST "http://localhost:4000/v1/evals/eval_123/cancel" \
|
||||
-H "Authorization: Bearer your-key"
|
||||
```
|
||||
|
||||
Returns: CancelEvalResponse with cancellation confirmation
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
select_data_generator,
|
||||
user_api_base,
|
||||
user_max_tokens,
|
||||
user_model,
|
||||
user_request_timeout,
|
||||
user_temperature,
|
||||
version,
|
||||
)
|
||||
|
||||
# Read request body (optional for cancel)
|
||||
body = await request.body()
|
||||
data = orjson.loads(body) if body else {}
|
||||
|
||||
# Set eval_id from path parameter
|
||||
data["eval_id"] = eval_id
|
||||
|
||||
# Extract model for routing (header > query > body)
|
||||
model = (
|
||||
data.get("model")
|
||||
or request.query_params.get("model")
|
||||
or request.headers.get("x-litellm-model")
|
||||
)
|
||||
if model:
|
||||
data["model"] = model
|
||||
|
||||
if "custom_llm_provider" not in data:
|
||||
data["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
# Process request using ProxyBaseLLMRequestProcessing
|
||||
processor = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
return await processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
route_type="acancel_eval",
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
llm_router=llm_router,
|
||||
general_settings=general_settings,
|
||||
proxy_config=proxy_config,
|
||||
select_data_generator=select_data_generator,
|
||||
model=data.get("model"),
|
||||
user_model=user_model,
|
||||
user_temperature=user_temperature,
|
||||
user_request_timeout=user_request_timeout,
|
||||
user_max_tokens=user_max_tokens,
|
||||
user_api_base=user_api_base,
|
||||
version=version,
|
||||
)
|
||||
except Exception as e:
|
||||
raise await processor._handle_llm_api_exception(
|
||||
e=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
version=version,
|
||||
)
|
||||
@@ -411,6 +411,7 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
|
||||
from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
|
||||
from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
|
||||
from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
|
||||
from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
|
||||
from litellm.proxy.openai_files_endpoints.files_endpoints import (
|
||||
router as openai_files_router,
|
||||
)
|
||||
@@ -12426,6 +12427,7 @@ app.include_router(llm_passthrough_router)
|
||||
app.include_router(mcp_management_router)
|
||||
app.include_router(anthropic_router)
|
||||
app.include_router(anthropic_skills_router)
|
||||
app.include_router(evals_router)
|
||||
app.include_router(claude_code_marketplace_router)
|
||||
app.include_router(google_router)
|
||||
app.include_router(langfuse_router)
|
||||
|
||||
@@ -73,6 +73,13 @@ ROUTE_ENDPOINT_MAPPING = {
|
||||
"aget_interaction": "/interactions/{interaction_id}",
|
||||
"adelete_interaction": "/interactions/{interaction_id}",
|
||||
"acancel_interaction": "/interactions/{interaction_id}/cancel",
|
||||
# OpenAI Evals API routes
|
||||
"acreate_eval": "/evals",
|
||||
"alist_evals": "/evals",
|
||||
"aget_eval": "/evals/{eval_id}",
|
||||
"aupdate_eval": "/evals/{eval_id}",
|
||||
"adelete_eval": "/evals/{eval_id}",
|
||||
"acancel_eval": "/evals/{eval_id}/cancel",
|
||||
}
|
||||
|
||||
|
||||
@@ -190,6 +197,12 @@ async def route_request(
|
||||
"acancel_interaction",
|
||||
"acancel_batch",
|
||||
"afile_delete",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
],
|
||||
):
|
||||
"""
|
||||
@@ -256,6 +269,36 @@ async def route_request(
|
||||
else:
|
||||
return getattr(litellm, f"{route_type}")(**data)
|
||||
elif llm_router is not None:
|
||||
# Evals API: always route to litellm directly (not through router)
|
||||
# But extract model credentials if a model is provided
|
||||
if route_type in [
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
]:
|
||||
# If a model is provided, get its credentials from the router
|
||||
model = data.get("model")
|
||||
if model and llm_router:
|
||||
try:
|
||||
# Try to get deployment credentials for this model
|
||||
deployment_creds = llm_router.get_deployment_credentials(model_id=model)
|
||||
if not deployment_creds:
|
||||
# Try by model group name
|
||||
deployment = llm_router.get_deployment_by_model_group_name(model_group_name=model)
|
||||
if deployment and deployment.litellm_params:
|
||||
deployment_creds = deployment.litellm_params.model_dump(exclude_none=True)
|
||||
|
||||
# If we found credentials, merge them into data (but don't override user-provided values)
|
||||
if deployment_creds:
|
||||
data.update(deployment_creds)
|
||||
except Exception:
|
||||
# If we can't get deployment creds, continue without them
|
||||
pass
|
||||
|
||||
return getattr(litellm, f"{route_type}")(**data)
|
||||
# Skip model-based routing for container operations
|
||||
if route_type in [
|
||||
"acreate_container",
|
||||
|
||||
Reference in New Issue
Block a user