mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-23 22:27:10 +00:00
Merge pull request #21375 from BerriAI/litellm_evals_api
[feat] Add support for Openai Evals API
This commit is contained in:
Sameer Kankute
@@ -0,0 +1,441 @@
|
||||
# /evals
|
||||
|
||||
LiteLLM Proxy supports OpenAI's Evaluations (Evals) API, allowing you to create, manage, and run evaluations to measure model performance against defined testing criteria.
|
||||
|
||||
## What are Evals?
|
||||
|
||||
OpenAI Evals API provides a structured way to:
|
||||
- **Create Evaluations**: Define testing criteria and data sources for evaluating model outputs
|
||||
- **Run Evaluations**: Execute evaluations against specific models and datasets
|
||||
- **Track Results**: Monitor evaluation progress and review detailed results
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Setup LiteLLM Proxy
|
||||
|
||||
First, start your LiteLLM Proxy server:
|
||||
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
|
||||
# Proxy will run on http://localhost:4000
|
||||
```
|
||||
|
||||
### Initialize OpenAI Client
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
# Point to your LiteLLM Proxy
|
||||
client = OpenAI(
|
||||
api_key="sk-1234", # Your LiteLLM proxy API key
|
||||
base_url="http://localhost:4000" # Your proxy URL
|
||||
)
|
||||
```
|
||||
|
||||
|
||||
For async operations:
|
||||
|
||||
```python
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
client = AsyncOpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Evaluation Management
|
||||
|
||||
### Create an Evaluation
|
||||
|
||||
Create an evaluation with testing criteria and data source configuration.
|
||||
|
||||
#### Example: Sentiment Classification Eval
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
# Create evaluation with label model grader
|
||||
eval_obj = client.evals.create(
|
||||
name="Sentiment Classification",
|
||||
data_source_config={
|
||||
"type": "stored_completions",
|
||||
"metadata": {"usecase": "chatbot"}
|
||||
},
|
||||
testing_criteria=[
|
||||
{
|
||||
"type": "label_model",
|
||||
"model": "gpt-4o-mini",
|
||||
"input": [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Classify the sentiment of the following statement as one of 'positive', 'neutral', or 'negative'"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Statement: {{item.input}}"
|
||||
}
|
||||
],
|
||||
"passing_labels": ["positive"],
|
||||
"labels": ["positive", "neutral", "negative"],
|
||||
"name": "Sentiment Grader"
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
# Note: If you want to use model-specific credentials for this evaluation, you can specify the model name in the extra body parameters.
|
||||
|
||||
print(f"Created eval: {eval_obj.id}")
|
||||
print(f"Eval name: {eval_obj.name}")
|
||||
```
|
||||
|
||||
#### Example: Push Notifications Summarizer Monitoring
|
||||
|
||||
This example shows how to monitor prompt changes for regressions in a push notifications summarizer:
|
||||
|
||||
```python
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
client = AsyncOpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
# Define data source for stored completions
|
||||
data_source_config = {
|
||||
"type": "stored_completions",
|
||||
"metadata": {
|
||||
"usecase": "push_notifications_summarizer"
|
||||
}
|
||||
}
|
||||
|
||||
# Define grader criteria
|
||||
GRADER_DEVELOPER_PROMPT = """
|
||||
Label the following push notification summary as either correct or incorrect.
|
||||
The push notification and the summary will be provided below.
|
||||
A good push notification summary is concise and snappy.
|
||||
If it is good, then label it as correct, if not, then incorrect.
|
||||
"""
|
||||
|
||||
GRADER_TEMPLATE_PROMPT = """
|
||||
Push notifications: {{item.input}}
|
||||
Summary: {{sample.output_text}}
|
||||
"""
|
||||
|
||||
push_notification_grader = {
|
||||
"name": "Push Notification Summary Grader",
|
||||
"type": "label_model",
|
||||
"model": "gpt-4o-mini",
|
||||
"input": [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": GRADER_DEVELOPER_PROMPT,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": GRADER_TEMPLATE_PROMPT,
|
||||
},
|
||||
],
|
||||
"passing_labels": ["correct"],
|
||||
"labels": ["correct", "incorrect"],
|
||||
}
|
||||
|
||||
# Create the evaluation
|
||||
eval_result = await client.evals.create(
|
||||
name="Push Notification Completion Monitoring",
|
||||
metadata={"description": "This eval monitors completions"},
|
||||
data_source_config=data_source_config,
|
||||
testing_criteria=[push_notification_grader],
|
||||
)
|
||||
|
||||
eval_id = eval_result.id
|
||||
print(f"Created eval: {eval_id}")
|
||||
```
|
||||
|
||||
### List Evaluations
|
||||
|
||||
Retrieve a list of all your evaluations with pagination support.
|
||||
|
||||
```python
|
||||
# List all evaluations
|
||||
evals_response = client.evals.list(
|
||||
limit=20,
|
||||
order="desc"
|
||||
)
|
||||
|
||||
for eval in evals_response.data:
|
||||
print(f"Eval ID: {eval.id}, Name: {eval.name}")
|
||||
|
||||
# Check if there are more evals
|
||||
if evals_response.has_more:
|
||||
# Fetch next page
|
||||
next_evals = client.evals.list(
|
||||
after=evals_response.last_id,
|
||||
limit=20
|
||||
)
|
||||
```
|
||||
|
||||
### Get a Specific Evaluation
|
||||
|
||||
Retrieve details of a specific evaluation by ID.
|
||||
|
||||
```python
|
||||
eval = client.evals.retrieve(
|
||||
eval_id="eval_abc123"
|
||||
)
|
||||
|
||||
print(f"Eval ID: {eval.id}")
|
||||
print(f"Name: {eval.name}")
|
||||
print(f"Data Source: {eval.data_source_config}")
|
||||
print(f"Testing Criteria: {eval.testing_criteria}")
|
||||
```
|
||||
|
||||
### Update an Evaluation
|
||||
|
||||
Update evaluation metadata or name.
|
||||
|
||||
```python
|
||||
updated_eval = client.evals.update(
|
||||
eval_id="eval_abc123",
|
||||
name="Updated Evaluation Name",
|
||||
metadata={
|
||||
"version": "2.0",
|
||||
"updated_by": "user@example.com"
|
||||
}
|
||||
)
|
||||
|
||||
print(f"Updated eval: {updated_eval.name}")
|
||||
```
|
||||
|
||||
### Delete an Evaluation
|
||||
|
||||
Permanently delete an evaluation.
|
||||
|
||||
```python
|
||||
delete_response = client.evals.delete(
|
||||
eval_id="eval_abc123"
|
||||
)
|
||||
|
||||
print(f"Deleted: {delete_response.deleted}") # True
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Evaluation Runs
|
||||
|
||||
### Create a Run
|
||||
|
||||
Execute an evaluation by creating a run. The run processes your data through the model and applies testing criteria.
|
||||
|
||||
#### Using Stored Completions
|
||||
|
||||
First, generate some test data by making chat completions with metadata:
|
||||
|
||||
```python
|
||||
from openai import AsyncOpenAI
|
||||
import asyncio
|
||||
|
||||
client = AsyncOpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
# Generate test data with different prompt versions
|
||||
push_notification_data = [
|
||||
"""
|
||||
- New message from Sarah: "Can you call me later?"
|
||||
- Your package has been delivered!
|
||||
- Flash sale: 20% off electronics for the next 2 hours!
|
||||
""",
|
||||
"""
|
||||
- Weather alert: Thunderstorm expected in your area.
|
||||
- Reminder: Doctor's appointment at 3 PM.
|
||||
- John liked your photo on Instagram.
|
||||
"""
|
||||
]
|
||||
|
||||
PROMPTS = [
|
||||
(
|
||||
"""
|
||||
You are a helpful assistant that summarizes push notifications.
|
||||
You are given a list of push notifications and you need to collapse them into a single one.
|
||||
Output only the final summary, nothing else.
|
||||
""",
|
||||
"v1"
|
||||
),
|
||||
(
|
||||
"""
|
||||
You are a helpful assistant that summarizes push notifications.
|
||||
You are given a list of push notifications and you need to collapse them into a single one.
|
||||
The summary should be longer than it needs to be and include more information than is necessary.
|
||||
Output only the final summary, nothing else.
|
||||
""",
|
||||
"v2"
|
||||
)
|
||||
]
|
||||
|
||||
# Create completions with metadata for tracking
|
||||
tasks = []
|
||||
for notifications in push_notification_data:
|
||||
for (prompt, version) in PROMPTS:
|
||||
tasks.append(client.chat.completions.create(
|
||||
model="gpt-4o-mini",
|
||||
messages=[
|
||||
{"role": "developer", "content": prompt},
|
||||
{"role": "user", "content": notifications},
|
||||
],
|
||||
metadata={
|
||||
"prompt_version": version,
|
||||
"usecase": "push_notifications_summarizer"
|
||||
}
|
||||
))
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
```
|
||||
|
||||
Now create runs to evaluate different prompt versions:
|
||||
|
||||
```python
|
||||
# Grade prompt_version=v1
|
||||
eval_run_result = await client.evals.runs.create(
|
||||
eval_id=eval_id,
|
||||
name="v1-run",
|
||||
data_source={
|
||||
"type": "completions",
|
||||
"source": {
|
||||
"type": "stored_completions",
|
||||
"metadata": {
|
||||
"prompt_version": "v1",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
print(f"Run ID: {eval_run_result.id}")
|
||||
print(f"Status: {eval_run_result.status}")
|
||||
print(f"Report URL: {eval_run_result.report_url}")
|
||||
|
||||
# Grade prompt_version=v2
|
||||
eval_run_result_v2 = await client.evals.runs.create(
|
||||
eval_id=eval_id,
|
||||
name="v2-run",
|
||||
data_source={
|
||||
"type": "completions",
|
||||
"source": {
|
||||
"type": "stored_completions",
|
||||
"metadata": {
|
||||
"prompt_version": "v2",
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
print(f"Run ID: {eval_run_result_v2.id}")
|
||||
print(f"Report URL: {eval_run_result_v2.report_url}")
|
||||
```
|
||||
|
||||
#### Using Completions with Different Models
|
||||
|
||||
Test how different models perform on the same inputs:
|
||||
|
||||
```python
|
||||
# Test with GPT-4o using stored completions as input
|
||||
tasks = []
|
||||
for prompt_version in ["v1", "v2"]:
|
||||
tasks.append(client.evals.runs.create(
|
||||
eval_id=eval_id,
|
||||
name=f"gpt-4o-run-{prompt_version}",
|
||||
data_source={
|
||||
"type": "completions",
|
||||
"input_messages": {
|
||||
"type": "item_reference",
|
||||
"item_reference": "item.input",
|
||||
},
|
||||
"model": "gpt-4o",
|
||||
"source": {
|
||||
"type": "stored_completions",
|
||||
"metadata": {
|
||||
"prompt_version": prompt_version,
|
||||
}
|
||||
}
|
||||
}
|
||||
))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
for run in results:
|
||||
print(f"Report URL: {run.report_url}")
|
||||
```
|
||||
|
||||
### List Runs
|
||||
|
||||
Get all runs for a specific evaluation.
|
||||
|
||||
```python
|
||||
# List all runs for an evaluation
|
||||
runs_response = client.evals.runs.list(
|
||||
eval_id="eval_abc123",
|
||||
limit=20,
|
||||
order="desc"
|
||||
)
|
||||
|
||||
for run in runs_response.data:
|
||||
print(f"Run ID: {run.id}")
|
||||
print(f"Status: {run.status}")
|
||||
print(f"Name: {run.name}")
|
||||
if run.result_counts:
|
||||
print(f"Results: {run.result_counts.passed}/{run.result_counts.total} passed")
|
||||
```
|
||||
|
||||
### Get Run Details
|
||||
|
||||
Retrieve detailed information about a specific run, including results.
|
||||
|
||||
```python
|
||||
run = client.evals.runs.retrieve(
|
||||
eval_id="eval_abc123",
|
||||
run_id="run_def456"
|
||||
)
|
||||
|
||||
print(f"Run ID: {run.id}")
|
||||
print(f"Status: {run.status}")
|
||||
print(f"Started: {run.started_at}")
|
||||
print(f"Completed: {run.completed_at}")
|
||||
|
||||
# Check results
|
||||
if run.result_counts:
|
||||
print(f"\nOverall Results:")
|
||||
print(f"Total: {run.result_counts.total}")
|
||||
print(f"Passed: {run.result_counts.passed}")
|
||||
print(f"Failed: {run.result_counts.failed}")
|
||||
print(f"Error: {run.result_counts.errored}")
|
||||
|
||||
# Per-criteria results
|
||||
if run.per_testing_criteria_results:
|
||||
for criteria_result in run.per_testing_criteria_results:
|
||||
print(f"\nCriteria {criteria_result.testing_criteria_index}:")
|
||||
print(f" Passed: {criteria_result.result_counts.passed}")
|
||||
print(f" Average Score: {criteria_result.average_score}")
|
||||
```
|
||||
|
||||
### Delete a Run
|
||||
|
||||
Permanently delete a run and its results.
|
||||
|
||||
```python
|
||||
delete_response = await client.evals.runs.delete(
|
||||
eval_id="eval_abc123",
|
||||
run_id="run_def456"
|
||||
)
|
||||
|
||||
print(f"Deleted: {delete_response.deleted}") # True
|
||||
print(f"Run ID: {delete_response.run_id}")
|
||||
```
|
||||
|
||||
@@ -573,6 +573,7 @@ const sidebars = {
|
||||
"proxy/managed_finetuning",
|
||||
]
|
||||
},
|
||||
"evals_api",
|
||||
"generateContent",
|
||||
"apply_guardrail",
|
||||
"bedrock_invoke",
|
||||
|
||||
@@ -1152,6 +1152,28 @@ from .skills.main import (
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .evals.main import (
|
||||
create_eval,
|
||||
acreate_eval,
|
||||
list_evals,
|
||||
alist_evals,
|
||||
get_eval,
|
||||
aget_eval,
|
||||
delete_eval,
|
||||
adelete_eval,
|
||||
cancel_eval,
|
||||
acancel_eval,
|
||||
create_run,
|
||||
acreate_run,
|
||||
list_runs,
|
||||
alist_runs,
|
||||
get_run,
|
||||
aget_run,
|
||||
delete_run,
|
||||
adelete_run,
|
||||
cancel_run,
|
||||
acancel_run,
|
||||
)
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
from .exceptions import (
|
||||
@@ -1732,6 +1754,37 @@ def __getattr__(name: str) -> Any:
|
||||
_globals["_service_logger"] = litellm._service_logger
|
||||
return _globals["_service_logger"]
|
||||
|
||||
# Lazy load evals module functions
|
||||
if name in ["acreate_eval", "alist_evals", "aget_eval", "aupdate_eval", "adelete_eval", "acancel_eval",
|
||||
"create_eval", "list_evals", "get_eval", "update_eval", "delete_eval", "cancel_eval",
|
||||
"acreate_run", "alist_runs", "aget_run", "acancel_run", "adelete_run",
|
||||
"create_run", "list_runs", "get_run", "cancel_run", "delete_run"]:
|
||||
from litellm.evals.main import (
|
||||
acreate_eval,
|
||||
alist_evals,
|
||||
aget_eval,
|
||||
aupdate_eval,
|
||||
adelete_eval,
|
||||
acancel_eval,
|
||||
create_eval,
|
||||
list_evals,
|
||||
get_eval,
|
||||
update_eval,
|
||||
delete_eval,
|
||||
cancel_eval,
|
||||
acreate_run,
|
||||
alist_runs,
|
||||
aget_run,
|
||||
acancel_run,
|
||||
adelete_run,
|
||||
create_run,
|
||||
list_runs,
|
||||
get_run,
|
||||
cancel_run,
|
||||
delete_run,
|
||||
)
|
||||
return locals()[name]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Evals API operations
|
||||
"""
|
||||
|
||||
from .main import (
|
||||
acancel_eval,
|
||||
acreate_eval,
|
||||
adelete_eval,
|
||||
aget_eval,
|
||||
alist_evals,
|
||||
aupdate_eval,
|
||||
cancel_eval,
|
||||
create_eval,
|
||||
delete_eval,
|
||||
get_eval,
|
||||
list_evals,
|
||||
update_eval,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"create_eval",
|
||||
"list_evals",
|
||||
"get_eval",
|
||||
"update_eval",
|
||||
"delete_eval",
|
||||
"cancel_eval",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Base configuration for Evals API
|
||||
"""
|
||||
|
||||
from .transformation import BaseEvalsAPIConfig
|
||||
|
||||
__all__ = ["BaseEvalsAPIConfig"]
|
||||
@@ -0,0 +1,542 @@
|
||||
"""
|
||||
Base configuration class for Evals API
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
CancelRunResponse,
|
||||
CreateEvalRequest,
|
||||
CreateRunRequest,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsParams,
|
||||
ListEvalsResponse,
|
||||
ListRunsParams,
|
||||
ListRunsResponse,
|
||||
Run,
|
||||
RunDeleteResponse,
|
||||
UpdateEvalRequest,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class BaseEvalsAPIConfig(ABC):
|
||||
"""Base configuration for Evals API providers"""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def custom_llm_provider(self) -> LlmProviders:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def validate_environment(
|
||||
self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
|
||||
) -> dict:
|
||||
"""
|
||||
Validate and update headers with provider-specific requirements
|
||||
|
||||
Args:
|
||||
headers: Base headers dictionary
|
||||
litellm_params: LiteLLM parameters
|
||||
|
||||
Returns:
|
||||
Updated headers dictionary
|
||||
"""
|
||||
return headers
|
||||
|
||||
@abstractmethod
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
endpoint: str,
|
||||
eval_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the complete URL for the API request
|
||||
|
||||
Args:
|
||||
api_base: Base API URL
|
||||
endpoint: API endpoint (e.g., 'evals', 'evals/{id}')
|
||||
eval_id: Optional eval ID for specific eval operations
|
||||
|
||||
Returns:
|
||||
Complete URL
|
||||
"""
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required")
|
||||
return f"{api_base}/v1/{endpoint}"
|
||||
|
||||
@abstractmethod
|
||||
def transform_create_eval_request(
|
||||
self,
|
||||
create_request: CreateEvalRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Dict:
|
||||
"""
|
||||
Transform create eval request to provider-specific format
|
||||
|
||||
Args:
|
||||
create_request: Eval creation parameters
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Provider-specific request body
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_create_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""
|
||||
Transform provider response to Eval object
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Eval object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_list_evals_request(
|
||||
self,
|
||||
list_params: ListEvalsParams,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform list evals request parameters
|
||||
|
||||
Args:
|
||||
list_params: List parameters (pagination, filters)
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, query_params)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_list_evals_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ListEvalsResponse:
|
||||
"""
|
||||
Transform provider response to ListEvalsResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
ListEvalsResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_get_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform get eval request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_get_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""
|
||||
Transform provider response to Eval object
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Eval object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_update_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
update_request: UpdateEvalRequest,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""
|
||||
Transform update eval request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
update_request: Update parameters
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers, body)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_update_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""
|
||||
Transform provider response to Eval object
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Eval object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_delete_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform delete eval request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_delete_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> DeleteEvalResponse:
|
||||
"""
|
||||
Transform provider response to DeleteEvalResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
DeleteEvalResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_cancel_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""
|
||||
Transform cancel eval request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers, body)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_cancel_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> CancelEvalResponse:
|
||||
"""
|
||||
Transform provider response to CancelEvalResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
CancelEvalResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
# Run API Transformations
|
||||
@abstractmethod
|
||||
def transform_create_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
create_request: CreateRunRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform create run request to provider-specific format
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
create_request: Run creation parameters
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, request_body)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_create_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Run:
|
||||
"""
|
||||
Transform provider response to Run object
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_list_runs_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
list_params: ListRunsParams,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform list runs request parameters
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
list_params: List parameters (pagination, filters)
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, query_params)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_list_runs_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ListRunsResponse:
|
||||
"""
|
||||
Transform provider response to ListRunsResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
ListRunsResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_get_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform get run request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
run_id: Run ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_get_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Run:
|
||||
"""
|
||||
Transform provider response to Run object
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
Run object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_cancel_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""
|
||||
Transform cancel run request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
run_id: Run ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers, body)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_cancel_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> CancelRunResponse:
|
||||
"""
|
||||
Transform provider response to CancelRunResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
CancelRunResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_delete_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""
|
||||
Transform delete run request
|
||||
|
||||
Args:
|
||||
eval_id: Eval ID
|
||||
run_id: Run ID
|
||||
api_base: Base API URL
|
||||
litellm_params: LiteLLM parameters
|
||||
headers: Request headers
|
||||
|
||||
Returns:
|
||||
Tuple of (url, headers, body)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def transform_delete_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> "RunDeleteResponse":
|
||||
"""
|
||||
Transform provider response to RunDeleteResponse
|
||||
|
||||
Args:
|
||||
raw_response: Raw HTTP response
|
||||
logging_obj: Logging object
|
||||
|
||||
Returns:
|
||||
RunDeleteResponse object
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict,
|
||||
) -> Exception:
|
||||
"""Get appropriate error class for the provider."""
|
||||
return BaseLLMException(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
OpenAI Evals API configuration
|
||||
"""
|
||||
|
||||
from .transformation import OpenAIEvalsConfig
|
||||
|
||||
__all__ = ["OpenAIEvalsConfig"]
|
||||
@@ -0,0 +1,426 @@
|
||||
"""
|
||||
OpenAI Evals API configuration and transformations
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.evals.transformation import (
|
||||
BaseEvalsAPIConfig,
|
||||
LiteLLMLoggingObj,
|
||||
)
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
CancelRunResponse,
|
||||
CreateEvalRequest,
|
||||
CreateRunRequest,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsParams,
|
||||
ListEvalsResponse,
|
||||
ListRunsParams,
|
||||
ListRunsResponse,
|
||||
Run,
|
||||
RunDeleteResponse,
|
||||
UpdateEvalRequest,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
||||
class OpenAIEvalsConfig(BaseEvalsAPIConfig):
|
||||
"""OpenAI-specific Evals API configuration"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.OPENAI
|
||||
|
||||
def validate_environment(
|
||||
self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
|
||||
) -> dict:
|
||||
"""Add OpenAI-specific headers"""
|
||||
import litellm
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
# Get API key following OpenAI pattern
|
||||
api_key = None
|
||||
if litellm_params:
|
||||
api_key = litellm_params.api_key
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.openai_key
|
||||
or get_secret_str("OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
if not api_key:
|
||||
raise ValueError("OPENAI_API_KEY is required for Evals API")
|
||||
|
||||
# Add required headers
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
endpoint: str,
|
||||
eval_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Get complete URL for OpenAI Evals API"""
|
||||
if api_base is None:
|
||||
api_base = "https://api.openai.com"
|
||||
|
||||
if eval_id:
|
||||
return f"{api_base}/v1/evals/{eval_id}"
|
||||
return f"{api_base}/v1/{endpoint}"
|
||||
|
||||
def transform_create_eval_request(
|
||||
self,
|
||||
create_request: CreateEvalRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Dict:
|
||||
"""Transform create eval request for OpenAI"""
|
||||
verbose_logger.debug("Transforming create eval request: %s", create_request)
|
||||
|
||||
# OpenAI expects the request body directly
|
||||
request_body = {k: v for k, v in create_request.items() if v is not None}
|
||||
|
||||
return request_body
|
||||
|
||||
def transform_create_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""Transform OpenAI response to Eval object"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming create eval response: %s", response_json)
|
||||
|
||||
return Eval(**response_json)
|
||||
|
||||
def transform_list_evals_request(
|
||||
self,
|
||||
list_params: ListEvalsParams,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform list evals request for OpenAI"""
|
||||
api_base = "https://api.openai.com"
|
||||
if litellm_params and litellm_params.api_base:
|
||||
api_base = litellm_params.api_base
|
||||
|
||||
url = self.get_complete_url(api_base=api_base, endpoint="evals")
|
||||
|
||||
# Build query parameters
|
||||
query_params: Dict[str, Any] = {}
|
||||
if "limit" in list_params and list_params["limit"]:
|
||||
query_params["limit"] = list_params["limit"]
|
||||
if "after" in list_params and list_params["after"]:
|
||||
query_params["after"] = list_params["after"]
|
||||
if "before" in list_params and list_params["before"]:
|
||||
query_params["before"] = list_params["before"]
|
||||
if "order" in list_params and list_params["order"]:
|
||||
query_params["order"] = list_params["order"]
|
||||
if "order_by" in list_params and list_params["order_by"]:
|
||||
query_params["order_by"] = list_params["order_by"]
|
||||
|
||||
verbose_logger.debug(
|
||||
"List evals request made to OpenAI Evals endpoint with params: %s",
|
||||
query_params,
|
||||
)
|
||||
|
||||
return url, query_params
|
||||
|
||||
def transform_list_evals_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ListEvalsResponse:
|
||||
"""Transform OpenAI response to ListEvalsResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming list evals response: %s", response_json)
|
||||
|
||||
return ListEvalsResponse(**response_json)
|
||||
|
||||
def transform_get_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform get eval request for OpenAI"""
|
||||
url = self.get_complete_url(
|
||||
api_base=api_base, endpoint="evals", eval_id=eval_id
|
||||
)
|
||||
|
||||
verbose_logger.debug("Get eval request - URL: %s", url)
|
||||
|
||||
return url, headers
|
||||
|
||||
def transform_get_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""Transform OpenAI response to Eval object"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming get eval response: %s", response_json)
|
||||
|
||||
return Eval(**response_json)
|
||||
|
||||
def transform_update_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
update_request: UpdateEvalRequest,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""Transform update eval request for OpenAI"""
|
||||
url = self.get_complete_url(
|
||||
api_base=api_base, endpoint="evals", eval_id=eval_id
|
||||
)
|
||||
|
||||
# Build request body
|
||||
request_body = {k: v for k, v in update_request.items() if v is not None}
|
||||
|
||||
verbose_logger.debug(
|
||||
"Update eval request - URL: %s, body: %s", url, request_body
|
||||
)
|
||||
|
||||
return url, headers, request_body
|
||||
|
||||
def transform_update_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Eval:
|
||||
"""Transform OpenAI response to Eval object"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming update eval response: %s", response_json)
|
||||
|
||||
return Eval(**response_json)
|
||||
|
||||
def transform_delete_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform delete eval request for OpenAI"""
|
||||
url = self.get_complete_url(
|
||||
api_base=api_base, endpoint="evals", eval_id=eval_id
|
||||
)
|
||||
|
||||
verbose_logger.debug("Delete eval request - URL: %s", url)
|
||||
|
||||
return url, headers
|
||||
|
||||
def transform_delete_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> DeleteEvalResponse:
|
||||
"""Transform OpenAI response to DeleteEvalResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming delete eval response: %s", response_json)
|
||||
|
||||
return DeleteEvalResponse(**response_json)
|
||||
|
||||
def transform_cancel_eval_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""Transform cancel eval request for OpenAI"""
|
||||
url = f"{self.get_complete_url(api_base=api_base, endpoint='evals', eval_id=eval_id)}/cancel"
|
||||
|
||||
# Empty body for cancel request
|
||||
request_body: Dict[str, Any] = {}
|
||||
|
||||
verbose_logger.debug("Cancel eval request - URL: %s", url)
|
||||
|
||||
return url, headers, request_body
|
||||
|
||||
def transform_cancel_eval_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> CancelEvalResponse:
|
||||
"""Transform OpenAI response to CancelEvalResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming cancel eval response: %s", response_json)
|
||||
|
||||
return CancelEvalResponse(**response_json)
|
||||
|
||||
# Run API Transformations
|
||||
def transform_create_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
create_request: CreateRunRequest,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform create run request for OpenAI"""
|
||||
api_base = "https://api.openai.com"
|
||||
if litellm_params and litellm_params.api_base:
|
||||
api_base = litellm_params.api_base
|
||||
|
||||
url = f"{api_base}/v1/evals/{eval_id}/runs"
|
||||
|
||||
# Build request body
|
||||
request_body = {k: v for k, v in create_request.items() if v is not None}
|
||||
|
||||
verbose_logger.debug(
|
||||
"Create run request - URL: %s, body: %s", url, request_body
|
||||
)
|
||||
|
||||
return url, request_body
|
||||
|
||||
def transform_create_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Run:
|
||||
"""Transform OpenAI response to Run object"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming create run response: %s", response_json)
|
||||
|
||||
return Run(**response_json)
|
||||
|
||||
def transform_list_runs_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
list_params: ListRunsParams,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform list runs request for OpenAI"""
|
||||
api_base = "https://api.openai.com"
|
||||
if litellm_params and litellm_params.api_base:
|
||||
api_base = litellm_params.api_base
|
||||
|
||||
url = f"{api_base}/v1/evals/{eval_id}/runs"
|
||||
|
||||
# Build query parameters
|
||||
query_params: Dict[str, Any] = {}
|
||||
if "limit" in list_params and list_params["limit"]:
|
||||
query_params["limit"] = list_params["limit"]
|
||||
if "after" in list_params and list_params["after"]:
|
||||
query_params["after"] = list_params["after"]
|
||||
if "before" in list_params and list_params["before"]:
|
||||
query_params["before"] = list_params["before"]
|
||||
if "order" in list_params and list_params["order"]:
|
||||
query_params["order"] = list_params["order"]
|
||||
|
||||
verbose_logger.debug(
|
||||
"List runs request made to OpenAI Evals endpoint with params: %s",
|
||||
query_params,
|
||||
)
|
||||
|
||||
return url, query_params
|
||||
|
||||
def transform_list_runs_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ListRunsResponse:
|
||||
"""Transform OpenAI response to ListRunsResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming list runs response: %s", response_json)
|
||||
|
||||
return ListRunsResponse(**response_json)
|
||||
|
||||
def transform_get_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""Transform get run request for OpenAI"""
|
||||
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
|
||||
|
||||
verbose_logger.debug("Get run request - URL: %s", url)
|
||||
|
||||
return url, headers
|
||||
|
||||
def transform_get_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> Run:
|
||||
"""Transform OpenAI response to Run object"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming get run response: %s", response_json)
|
||||
|
||||
return Run(**response_json)
|
||||
|
||||
def transform_cancel_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""Transform cancel run request for OpenAI"""
|
||||
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
|
||||
|
||||
# Empty body for cancel request
|
||||
request_body: Dict[str, Any] = {}
|
||||
|
||||
verbose_logger.debug("Cancel run request - URL: %s", url)
|
||||
|
||||
return url, headers, request_body
|
||||
|
||||
def transform_cancel_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> CancelRunResponse:
|
||||
"""Transform OpenAI response to CancelRunResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming cancel run response: %s", response_json)
|
||||
|
||||
return CancelRunResponse(**response_json)
|
||||
|
||||
def transform_delete_run_request(
|
||||
self,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict, Dict]:
|
||||
"""Transform delete run request for OpenAI"""
|
||||
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
|
||||
|
||||
# Empty body for delete request
|
||||
request_body: Dict[str, Any] = {}
|
||||
|
||||
verbose_logger.debug("Delete run request - URL: %s", url)
|
||||
|
||||
return url, headers, request_body
|
||||
|
||||
def transform_delete_run_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> RunDeleteResponse:
|
||||
"""Transform OpenAI response to RunDeleteResponse"""
|
||||
response_json = raw_response.json()
|
||||
verbose_logger.debug("Transforming delete run response: %s", response_json)
|
||||
|
||||
return RunDeleteResponse(**response_json)
|
||||
@@ -526,6 +526,17 @@ class ProxyBaseLLMRequestProcessing:
|
||||
"acancel_interaction",
|
||||
"asend_message",
|
||||
"call_mcp_tool",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
],
|
||||
version: Optional[str] = None,
|
||||
user_model: Optional[str] = None,
|
||||
@@ -708,6 +719,17 @@ class ProxyBaseLLMRequestProcessing:
|
||||
"acancel_interaction",
|
||||
"acancel_batch",
|
||||
"afile_delete",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
],
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
general_settings: dict,
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
OpenAI Evals API endpoints
|
||||
"""
|
||||
|
||||
from .endpoints import router
|
||||
|
||||
__all__ = ["router"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -411,6 +411,7 @@ from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
|
||||
from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
|
||||
from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
|
||||
from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
|
||||
from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
|
||||
from litellm.proxy.openai_files_endpoints.files_endpoints import (
|
||||
router as openai_files_router,
|
||||
)
|
||||
@@ -12426,6 +12427,7 @@ app.include_router(llm_passthrough_router)
|
||||
app.include_router(mcp_management_router)
|
||||
app.include_router(anthropic_router)
|
||||
app.include_router(anthropic_skills_router)
|
||||
app.include_router(evals_router)
|
||||
app.include_router(claude_code_marketplace_router)
|
||||
app.include_router(google_router)
|
||||
app.include_router(langfuse_router)
|
||||
|
||||
@@ -73,6 +73,19 @@ ROUTE_ENDPOINT_MAPPING = {
|
||||
"aget_interaction": "/interactions/{interaction_id}",
|
||||
"adelete_interaction": "/interactions/{interaction_id}",
|
||||
"acancel_interaction": "/interactions/{interaction_id}/cancel",
|
||||
# OpenAI Evals API routes
|
||||
"acreate_eval": "/evals",
|
||||
"alist_evals": "/evals",
|
||||
"aget_eval": "/evals/{eval_id}",
|
||||
"aupdate_eval": "/evals/{eval_id}",
|
||||
"adelete_eval": "/evals/{eval_id}",
|
||||
"acancel_eval": "/evals/{eval_id}/cancel",
|
||||
# OpenAI Evals Runs API routes
|
||||
"acreate_run": "/evals/{eval_id}/runs",
|
||||
"alist_runs": "/evals/{eval_id}/runs",
|
||||
"aget_run": "/evals/{eval_id}/runs/{run_id}",
|
||||
"acancel_run": "/evals/{eval_id}/runs/{run_id}/cancel",
|
||||
"adelete_run": "/evals/{eval_id}/runs/{run_id}",
|
||||
}
|
||||
|
||||
|
||||
@@ -190,6 +203,17 @@ async def route_request(
|
||||
"acancel_interaction",
|
||||
"acancel_batch",
|
||||
"afile_delete",
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
],
|
||||
):
|
||||
"""
|
||||
@@ -256,6 +280,41 @@ async def route_request(
|
||||
else:
|
||||
return getattr(litellm, f"{route_type}")(**data)
|
||||
elif llm_router is not None:
|
||||
# Evals API: always route to litellm directly (not through router)
|
||||
# But extract model credentials if a model is provided
|
||||
if route_type in [
|
||||
"acreate_eval",
|
||||
"alist_evals",
|
||||
"aget_eval",
|
||||
"aupdate_eval",
|
||||
"adelete_eval",
|
||||
"acancel_eval",
|
||||
"acreate_run",
|
||||
"alist_runs",
|
||||
"aget_run",
|
||||
"acancel_run",
|
||||
"adelete_run",
|
||||
]:
|
||||
# If a model is provided, get its credentials from the router
|
||||
model = data.get("model")
|
||||
if model and llm_router:
|
||||
try:
|
||||
# Try to get deployment credentials for this model
|
||||
deployment_creds = llm_router.get_deployment_credentials(model_id=model)
|
||||
if not deployment_creds:
|
||||
# Try by model group name
|
||||
deployment = llm_router.get_deployment_by_model_group_name(model_group_name=model)
|
||||
if deployment and deployment.litellm_params:
|
||||
deployment_creds = deployment.litellm_params.model_dump(exclude_none=True)
|
||||
|
||||
# If we found credentials, merge them into data (but don't override user-provided values)
|
||||
if deployment_creds:
|
||||
data.update(deployment_creds)
|
||||
except Exception:
|
||||
# If we can't get deployment creds, continue without them
|
||||
pass
|
||||
|
||||
return getattr(litellm, f"{route_type}")(**data)
|
||||
# Skip model-based routing for container operations
|
||||
if route_type in [
|
||||
"acreate_container",
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
"""
|
||||
Type definitions for OpenAI Evals API
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
|
||||
# Evals API Request Types
|
||||
class DataSourceConfigCustom(TypedDict, total=False):
|
||||
"""Data source configuration for custom data sources"""
|
||||
|
||||
type: Required[Literal["custom"]]
|
||||
"""Data source type - custom"""
|
||||
|
||||
item_schema: Required[Dict[str, Any]]
|
||||
"""JSON schema describing the structure of each row in the dataset"""
|
||||
|
||||
include_sample_schema: Optional[bool]
|
||||
"""Whether eval expects sample schema population"""
|
||||
|
||||
|
||||
class DataSourceConfigLogs(TypedDict, total=False):
|
||||
"""Data source configuration for logs-based evals"""
|
||||
|
||||
type: Required[Literal["logs"]]
|
||||
"""Data source type - logs"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]]
|
||||
"""Optional metadata for filtering logs"""
|
||||
|
||||
|
||||
class DataSourceConfigStoredCompletions(TypedDict, total=False):
|
||||
"""Data source configuration for stored completions (deprecated)"""
|
||||
|
||||
type: Required[Literal["stored_completions"]]
|
||||
"""Data source type - stored_completions (deprecated)"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]]
|
||||
"""Optional metadata for filtering stored completions"""
|
||||
|
||||
|
||||
DataSourceConfig = Union[DataSourceConfigCustom, DataSourceConfigLogs, DataSourceConfigStoredCompletions]
|
||||
|
||||
|
||||
class LLMAsJudgeGraderConfig(TypedDict, total=False):
|
||||
"""Configuration for LLM as judge grading"""
|
||||
|
||||
type: Required[Literal["llm_as_judge"]]
|
||||
"""Grader type - LLM as judge"""
|
||||
|
||||
model: Optional[str]
|
||||
"""Model to use as judge (e.g., 'gpt-4')"""
|
||||
|
||||
prompt: Optional[str]
|
||||
"""Custom prompt for the judge model"""
|
||||
|
||||
|
||||
class GroundTruthGraderConfig(TypedDict, total=False):
|
||||
"""Configuration for ground truth grading"""
|
||||
|
||||
type: Required[Literal["ground_truth"]]
|
||||
"""Grader type - ground truth comparison"""
|
||||
|
||||
metric: Optional[Literal["exact_match", "f1_score", "bleu"]]
|
||||
"""Metric to use for comparison"""
|
||||
|
||||
|
||||
class CustomGraderConfig(TypedDict, total=False):
|
||||
"""Configuration for custom grading function"""
|
||||
|
||||
type: Required[Literal["custom"]]
|
||||
"""Grader type - custom"""
|
||||
|
||||
function_id: Required[str]
|
||||
"""ID of the custom grading function"""
|
||||
|
||||
|
||||
GraderConfig = Union[LLMAsJudgeGraderConfig, GroundTruthGraderConfig, CustomGraderConfig]
|
||||
|
||||
|
||||
class CreateEvalRequest(TypedDict, total=False):
|
||||
"""Request parameters for creating an evaluation"""
|
||||
|
||||
name: Optional[str]
|
||||
"""The name of the evaluation"""
|
||||
|
||||
data_source_config: Required[DataSourceConfig]
|
||||
"""Configuration for the data source"""
|
||||
|
||||
testing_criteria: Required[List[GraderConfig]]
|
||||
"""List of graders for all eval runs"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]]
|
||||
"""Set of 16 key-value pairs that can be attached to an object (max 64 char keys, 512 char values)"""
|
||||
|
||||
|
||||
class UpdateEvalRequest(TypedDict, total=False):
|
||||
"""Request parameters for updating an evaluation"""
|
||||
|
||||
name: Optional[str]
|
||||
"""Updated name"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]]
|
||||
"""Updated metadata"""
|
||||
|
||||
|
||||
class ListEvalsParams(TypedDict, total=False):
|
||||
"""Query parameters for listing evaluations"""
|
||||
|
||||
limit: Optional[int]
|
||||
"""Number of results to return per page. Maximum value is 100. Defaults to 20."""
|
||||
|
||||
after: Optional[str]
|
||||
"""Cursor for pagination - returns evals after this ID"""
|
||||
|
||||
before: Optional[str]
|
||||
"""Cursor for pagination - returns evals before this ID"""
|
||||
|
||||
order: Optional[Literal["asc", "desc"]]
|
||||
"""Sort order for results. Defaults to 'desc'."""
|
||||
|
||||
order_by: Optional[Literal["created_at", "updated_at"]]
|
||||
"""Field to sort by. Defaults to 'created_at'."""
|
||||
|
||||
|
||||
# Evals API Response Types
|
||||
class Eval(BaseModel):
|
||||
"""Represents an evaluation from the OpenAI Evals API"""
|
||||
|
||||
id: str
|
||||
"""Unique identifier for the evaluation"""
|
||||
|
||||
object: str = "eval"
|
||||
"""Object type, always 'eval'"""
|
||||
|
||||
created_at: int
|
||||
"""Unix timestamp of when the evaluation was created"""
|
||||
|
||||
updated_at: Optional[int] = None
|
||||
"""Unix timestamp of when the evaluation was last updated"""
|
||||
|
||||
name: Optional[str] = None
|
||||
"""The name of the evaluation"""
|
||||
|
||||
data_source_config: Dict[str, Any]
|
||||
"""Configuration for the data source"""
|
||||
|
||||
testing_criteria: List[Dict[str, Any]]
|
||||
"""List of graders for the evaluation"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
"""Additional metadata"""
|
||||
|
||||
|
||||
class ListEvalsResponse(BaseModel):
|
||||
"""Response from listing evaluations"""
|
||||
|
||||
object: str = "list"
|
||||
"""Object type, always 'list'"""
|
||||
|
||||
data: List[Eval]
|
||||
"""List of evaluations"""
|
||||
|
||||
first_id: Optional[str] = None
|
||||
"""ID of the first evaluation in the list"""
|
||||
|
||||
last_id: Optional[str] = None
|
||||
"""ID of the last evaluation in the list"""
|
||||
|
||||
has_more: bool = False
|
||||
"""Whether there are more evaluations available"""
|
||||
|
||||
|
||||
class DeleteEvalResponse(BaseModel):
|
||||
"""Response from deleting an evaluation"""
|
||||
|
||||
eval_id: str
|
||||
"""The ID of the deleted evaluation"""
|
||||
|
||||
object: str = "eval.deleted"
|
||||
"""Object type, always 'eval.deleted'"""
|
||||
|
||||
deleted: bool
|
||||
"""Whether the evaluation was successfully deleted"""
|
||||
|
||||
|
||||
class CancelEvalResponse(BaseModel):
|
||||
"""Response from cancelling an evaluation"""
|
||||
|
||||
id: str
|
||||
"""The ID of the cancelled evaluation"""
|
||||
|
||||
object: str = "eval"
|
||||
"""Object type, always 'eval'"""
|
||||
|
||||
status: Literal["cancelled"]
|
||||
"""Status after cancellation, always 'cancelled'"""
|
||||
|
||||
|
||||
# Run API Request Types
|
||||
class DataSourceDatasetConfig(TypedDict, total=False):
|
||||
"""Data source configuration for dataset-based runs"""
|
||||
|
||||
type: Required[Literal["dataset"]]
|
||||
"""Data source type - dataset"""
|
||||
|
||||
dataset_id: Required[str]
|
||||
"""ID of the dataset to use for the run"""
|
||||
|
||||
|
||||
class DataSourceSampleSetConfig(TypedDict, total=False):
|
||||
"""Data source configuration for sample set-based runs"""
|
||||
|
||||
type: Required[Literal["sample_set"]]
|
||||
"""Data source type - sample_set"""
|
||||
|
||||
sample_set_id: Required[str]
|
||||
"""ID of the sample set to use for the run"""
|
||||
|
||||
|
||||
class DataSourceInlineConfig(TypedDict, total=False):
|
||||
"""Data source configuration for inline samples"""
|
||||
|
||||
type: Required[Literal["inline"]]
|
||||
"""Data source type - inline"""
|
||||
|
||||
samples: Required[List[Dict[str, Any]]]
|
||||
"""List of inline samples to use for the run"""
|
||||
|
||||
|
||||
RunDataSourceConfig = Union[
|
||||
DataSourceDatasetConfig, DataSourceSampleSetConfig, DataSourceInlineConfig
|
||||
]
|
||||
|
||||
|
||||
class CompletionConfig(TypedDict, total=False):
|
||||
"""Configuration for model completions in a run"""
|
||||
|
||||
model: Required[str]
|
||||
"""Model to use for completions"""
|
||||
|
||||
temperature: Optional[float]
|
||||
"""Sampling temperature (0-2)"""
|
||||
|
||||
max_tokens: Optional[int]
|
||||
"""Maximum tokens to generate"""
|
||||
|
||||
top_p: Optional[float]
|
||||
"""Nucleus sampling parameter"""
|
||||
|
||||
frequency_penalty: Optional[float]
|
||||
"""Frequency penalty (-2.0 to 2.0)"""
|
||||
|
||||
presence_penalty: Optional[float]
|
||||
"""Presence penalty (-2.0 to 2.0)"""
|
||||
|
||||
|
||||
class CreateRunRequest(TypedDict, total=False):
|
||||
"""Request parameters for creating a run"""
|
||||
|
||||
data_source: Required[Dict[str, Any]]
|
||||
"""Data source configuration for the run (can be jsonl, completions, or responses type)"""
|
||||
|
||||
name: Optional[str]
|
||||
"""Optional name for the run"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]]
|
||||
"""Optional metadata for the run"""
|
||||
|
||||
|
||||
class ListRunsParams(TypedDict, total=False):
|
||||
"""Query parameters for listing runs"""
|
||||
|
||||
limit: Optional[int]
|
||||
"""Number of results to return per page. Maximum value is 100. Defaults to 20."""
|
||||
|
||||
after: Optional[str]
|
||||
"""Cursor for pagination - returns runs after this ID"""
|
||||
|
||||
before: Optional[str]
|
||||
"""Cursor for pagination - returns runs before this ID"""
|
||||
|
||||
order: Optional[Literal["asc", "desc"]]
|
||||
"""Sort order for results. Defaults to 'desc'."""
|
||||
|
||||
|
||||
# Run API Response Types
|
||||
class ResultCounts(BaseModel):
|
||||
"""Result counts for a run"""
|
||||
|
||||
total: int
|
||||
"""Total number of results"""
|
||||
|
||||
passed: int = 0
|
||||
"""Number of passed results"""
|
||||
|
||||
failed: int = 0
|
||||
"""Number of failed results"""
|
||||
|
||||
error: int = 0
|
||||
"""Number of error results"""
|
||||
|
||||
|
||||
class PerTestingCriteriaResult(BaseModel):
|
||||
"""Results for a specific testing criteria"""
|
||||
|
||||
testing_criteria_index: int
|
||||
"""Index of the testing criteria"""
|
||||
|
||||
result_counts: ResultCounts
|
||||
"""Result counts for this criteria"""
|
||||
|
||||
average_score: Optional[float] = None
|
||||
"""Average score for this criteria"""
|
||||
|
||||
|
||||
class Run(BaseModel):
|
||||
"""Represents a run from the OpenAI Evals API"""
|
||||
|
||||
id: str
|
||||
"""Unique identifier for the run"""
|
||||
|
||||
object: str = "eval.run"
|
||||
"""Object type, always 'eval.run'"""
|
||||
|
||||
created_at: int
|
||||
"""Unix timestamp of when the run was created"""
|
||||
|
||||
status: Literal["queued", "running", "completed", "failed", "cancelled"]
|
||||
"""Current status of the run"""
|
||||
|
||||
data_source: Dict[str, Any]
|
||||
"""Data source configuration used for the run"""
|
||||
|
||||
eval_id: str
|
||||
"""ID of the evaluation this run belongs to"""
|
||||
|
||||
name: Optional[str] = None
|
||||
"""Name of the run"""
|
||||
|
||||
started_at: Optional[int] = None
|
||||
"""Unix timestamp of when the run started"""
|
||||
|
||||
completed_at: Optional[int] = None
|
||||
"""Unix timestamp of when the run completed"""
|
||||
|
||||
model: Optional[str] = None
|
||||
"""Model used for the run, if any"""
|
||||
|
||||
per_model_usage: Optional[Any] = None
|
||||
"""Model usage details per model, if available"""
|
||||
|
||||
per_testing_criteria_results: Optional[List[PerTestingCriteriaResult]] = None
|
||||
"""Per-criteria results"""
|
||||
|
||||
report_url: Optional[str] = None
|
||||
"""URL for the evaluation report"""
|
||||
|
||||
result_counts: Optional[Dict[str, int]] = None
|
||||
"""Aggregate result counts (e.g., {"passed": 0, "failed": 0, "errored": 0, "total": 0})"""
|
||||
|
||||
shared_with_openai: Optional[bool] = None
|
||||
"""Whether run is shared with OpenAI"""
|
||||
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
"""Additional metadata"""
|
||||
|
||||
error: Optional[Dict[str, Any]] = None
|
||||
"""Error details if the run failed"""
|
||||
|
||||
|
||||
class ListRunsResponse(BaseModel):
|
||||
"""Response from listing runs"""
|
||||
|
||||
object: str = "list"
|
||||
"""Object type, always 'list'"""
|
||||
|
||||
data: List[Run]
|
||||
"""List of runs"""
|
||||
|
||||
first_id: Optional[str] = None
|
||||
"""ID of the first run in the list"""
|
||||
|
||||
last_id: Optional[str] = None
|
||||
"""ID of the last run in the list"""
|
||||
|
||||
has_more: bool = False
|
||||
"""Whether there are more runs available"""
|
||||
|
||||
|
||||
class CancelRunResponse(BaseModel):
|
||||
"""Response from cancelling a run"""
|
||||
|
||||
id: str
|
||||
"""The ID of the cancelled run"""
|
||||
|
||||
object: str = "eval.run"
|
||||
"""Object type, always 'eval.run'"""
|
||||
|
||||
status: Literal["cancelled"]
|
||||
"""Status after cancellation, always 'cancelled'"""
|
||||
|
||||
|
||||
class RunDeleteResponse(BaseModel):
|
||||
"""Response from deleting a run"""
|
||||
|
||||
run_id: str
|
||||
"""The ID of the deleted run"""
|
||||
|
||||
object: Optional[str] = "eval.run.deleted"
|
||||
"""Object type, always 'eval.run.deleted'"""
|
||||
|
||||
deleted: Optional[bool] = True
|
||||
"""Whether the run was successfully deleted"""
|
||||
@@ -380,6 +380,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig
|
||||
from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig
|
||||
|
||||
@@ -8282,6 +8283,25 @@ class ProviderConfigManager:
|
||||
return litellm.AnthropicSkillsConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_evals_api_config(
|
||||
provider: LlmProviders,
|
||||
) -> Optional["BaseEvalsAPIConfig"]:
|
||||
"""
|
||||
Get provider-specific Evals API configuration
|
||||
|
||||
Args:
|
||||
provider: The LLM provider
|
||||
|
||||
Returns:
|
||||
Provider-specific Evals API config or None
|
||||
"""
|
||||
if litellm.LlmProviders.OPENAI == provider:
|
||||
from litellm.llms.openai.evals.transformation import OpenAIEvalsConfig
|
||||
|
||||
return OpenAIEvalsConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_text_completion_config(
|
||||
model: str,
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""
|
||||
Tests for Evals API operations across providers
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
import litellm
|
||||
from litellm.types.llms.openai_evals import (
|
||||
CancelEvalResponse,
|
||||
DeleteEvalResponse,
|
||||
Eval,
|
||||
ListEvalsResponse,
|
||||
)
|
||||
|
||||
|
||||
class BaseEvalsAPITest(ABC):
|
||||
"""
|
||||
Base test class for Evals API operations.
|
||||
Tests create, list, get, update, delete, and cancel operations.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_custom_llm_provider(self) -> str:
|
||||
"""Return the provider name (e.g., 'openai')"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_api_key(self) -> Optional[str]:
|
||||
"""Return the API key for the provider"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_api_base(self) -> Optional[str]:
|
||||
"""Return the API base URL for the provider"""
|
||||
pass
|
||||
|
||||
def test_create_eval(self):
|
||||
"""
|
||||
Test creating an evaluation.
|
||||
"""
|
||||
import time
|
||||
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
# Create eval with stored_completions data source
|
||||
unique_name = f"Test Eval {int(time.time())}"
|
||||
|
||||
response = litellm.create_eval(
|
||||
name=unique_name,
|
||||
data_source_config={
|
||||
"type": "stored_completions",
|
||||
"metadata": {"usecase": "chatbot"},
|
||||
},
|
||||
testing_criteria=[
|
||||
{
|
||||
"type": "label_model",
|
||||
"model": "gpt-4o",
|
||||
"input": [
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Classify the sentiment as 'positive' or 'negative'",
|
||||
},
|
||||
{"role": "user", "content": "Statement: {{item.input}}"},
|
||||
],
|
||||
"passing_labels": ["positive"],
|
||||
"labels": ["positive", "negative"],
|
||||
"name": "Sentiment grader",
|
||||
}
|
||||
],
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, Eval)
|
||||
assert response.id is not None
|
||||
assert response.name == unique_name
|
||||
print(f"Created eval: {response}")
|
||||
print(f"Eval ID: {response.id}")
|
||||
|
||||
def test_list_evals(self):
|
||||
"""
|
||||
Test listing evaluations.
|
||||
"""
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
response = litellm.list_evals(
|
||||
limit=10,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, ListEvalsResponse)
|
||||
assert hasattr(response, "data")
|
||||
assert hasattr(response, "has_more")
|
||||
print(f"Listed evals: {len(response.data)} evaluations")
|
||||
|
||||
def test_get_eval(self):
|
||||
"""
|
||||
Test getting a specific evaluation by ID.
|
||||
"""
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
# First list existing evals to get an ID
|
||||
list_response = litellm.list_evals(
|
||||
limit=1,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert isinstance(list_response, ListEvalsResponse)
|
||||
|
||||
if list_response.data and len(list_response.data) > 0:
|
||||
eval_id = list_response.data[0].id
|
||||
print(f"Testing with eval ID: {eval_id}")
|
||||
|
||||
# Get the eval
|
||||
response = litellm.get_eval(
|
||||
eval_id=eval_id,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, Eval)
|
||||
assert response.id == eval_id
|
||||
print(f"Retrieved eval: {response}")
|
||||
else:
|
||||
pytest.skip("No existing evals to test with")
|
||||
|
||||
def test_update_eval(self):
|
||||
"""
|
||||
Test updating an evaluation.
|
||||
"""
|
||||
import time
|
||||
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
# First list existing evals
|
||||
list_response = litellm.list_evals(
|
||||
limit=1,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert isinstance(list_response, ListEvalsResponse)
|
||||
|
||||
if list_response.data and len(list_response.data) > 0:
|
||||
eval_id = list_response.data[0].id
|
||||
updated_name = f"Updated Eval {int(time.time())}"
|
||||
|
||||
# Update the eval
|
||||
response = litellm.update_eval(
|
||||
eval_id=eval_id,
|
||||
name=updated_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert isinstance(response, Eval)
|
||||
assert response.id == eval_id
|
||||
assert response.name == updated_name
|
||||
print(f"Updated eval: {response}")
|
||||
else:
|
||||
pytest.skip("No existing evals to test with")
|
||||
|
||||
def test_delete_eval(self):
|
||||
"""
|
||||
Test deleting an evaluation.
|
||||
"""
|
||||
custom_llm_provider = self.get_custom_llm_provider()
|
||||
api_key = self.get_api_key()
|
||||
api_base = self.get_api_base()
|
||||
|
||||
if not api_key:
|
||||
pytest.skip(f"No API key provided for {custom_llm_provider}")
|
||||
|
||||
# Skip this test to avoid deleting production evals
|
||||
pytest.skip("Skipping delete test to preserve existing evals")
|
||||
|
||||
|
||||
class TestOpenAIEvalsAPI(BaseEvalsAPITest):
|
||||
"""
|
||||
Test OpenAI Evals API implementation.
|
||||
"""
|
||||
|
||||
def get_custom_llm_provider(self) -> str:
|
||||
return "openai"
|
||||
|
||||
def get_api_key(self) -> Optional[str]:
|
||||
return os.environ.get("OPENAI_API_KEY")
|
||||
|
||||
def get_api_base(self) -> Optional[str]:
|
||||
return os.environ.get("OPENAI_API_BASE")
|
||||
@@ -0,0 +1 @@
|
||||
"""OpenAI Evals API tests"""
|
||||
@@ -0,0 +1,257 @@
|
||||
"""
|
||||
Unit tests for OpenAI Evals API transformation
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.llms.openai.evals.transformation import OpenAIEvalsConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def config() -> OpenAIEvalsConfig:
|
||||
return OpenAIEvalsConfig()
|
||||
|
||||
|
||||
def test_validate_environment_sets_headers(config: OpenAIEvalsConfig):
|
||||
"""Test that validate_environment correctly sets authorization headers"""
|
||||
headers: dict = {}
|
||||
params = GenericLiteLLMParams(api_key="sk-test-12345")
|
||||
|
||||
result = config.validate_environment(headers=headers, litellm_params=params)
|
||||
|
||||
assert result["Authorization"] == "Bearer sk-test-12345"
|
||||
assert result["Content-Type"] == "application/json"
|
||||
|
||||
|
||||
def test_validate_environment_requires_api_key(config: OpenAIEvalsConfig, monkeypatch):
|
||||
"""Test that validate_environment raises error when no API key is provided"""
|
||||
import os
|
||||
|
||||
# Ensure OPENAI_API_KEY environment variable is None before validation
|
||||
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
||||
|
||||
headers: dict = {}
|
||||
params = GenericLiteLLMParams()
|
||||
|
||||
with pytest.raises(ValueError, match="OPENAI_API_KEY is required"):
|
||||
config.validate_environment(headers=headers, litellm_params=params)
|
||||
|
||||
|
||||
def test_get_complete_url_with_eval_id(config: OpenAIEvalsConfig):
|
||||
"""Test URL construction with eval_id"""
|
||||
url = config.get_complete_url(
|
||||
api_base="https://api.openai.com",
|
||||
endpoint="evals",
|
||||
eval_id="eval_123",
|
||||
)
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123"
|
||||
|
||||
|
||||
def test_get_complete_url_without_eval_id(config: OpenAIEvalsConfig):
|
||||
"""Test URL construction without eval_id"""
|
||||
url = config.get_complete_url(
|
||||
api_base="https://api.openai.com",
|
||||
endpoint="evals",
|
||||
)
|
||||
assert url == "https://api.openai.com/v1/evals"
|
||||
|
||||
|
||||
def test_transform_create_eval_request(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of create eval request"""
|
||||
create_request = {
|
||||
"name": "Test Eval",
|
||||
"data_source_config": {
|
||||
"type": "stored_completions",
|
||||
"metadata": {"usecase": "chatbot"}
|
||||
},
|
||||
"testing_criteria": [
|
||||
{
|
||||
"type": "label_model",
|
||||
"model": "gpt-4o",
|
||||
"input": [{"role": "user", "content": "Test"}],
|
||||
"passing_labels": ["positive"],
|
||||
"labels": ["positive", "negative"],
|
||||
"name": "Test Grader"
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
result = config.transform_create_eval_request(
|
||||
create_request=create_request,
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["name"] == "Test Eval"
|
||||
assert result["data_source_config"]["type"] == "stored_completions"
|
||||
assert len(result["testing_criteria"]) == 1
|
||||
assert result["testing_criteria"][0]["type"] == "label_model"
|
||||
|
||||
|
||||
def test_transform_create_eval_response(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of create eval response"""
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "eval_123",
|
||||
"object": "eval",
|
||||
"created_at": 1234567890,
|
||||
"name": "Test Eval",
|
||||
"data_source_config": {"type": "stored_completions"},
|
||||
"testing_criteria": [],
|
||||
},
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/evals"),
|
||||
)
|
||||
|
||||
result = config.transform_create_eval_response(
|
||||
raw_response=response,
|
||||
logging_obj=None, # type: ignore
|
||||
)
|
||||
|
||||
assert result.id == "eval_123"
|
||||
assert result.object == "eval"
|
||||
assert result.name == "Test Eval"
|
||||
|
||||
|
||||
def test_transform_list_evals_request(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of list evals request"""
|
||||
list_params = {
|
||||
"limit": 10,
|
||||
"after": "eval_123",
|
||||
"order": "desc",
|
||||
}
|
||||
|
||||
url, query_params = config.transform_list_evals_request(
|
||||
list_params=list_params,
|
||||
litellm_params=GenericLiteLLMParams(api_base="https://api.openai.com"),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals"
|
||||
assert query_params["limit"] == 10
|
||||
assert query_params["after"] == "eval_123"
|
||||
assert query_params["order"] == "desc"
|
||||
|
||||
|
||||
def test_transform_list_evals_response(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of list evals response"""
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "eval_123",
|
||||
"object": "eval",
|
||||
"created_at": 1234567890,
|
||||
"name": "Test Eval",
|
||||
"data_source_config": {"type": "stored_completions"},
|
||||
"testing_criteria": [],
|
||||
}
|
||||
],
|
||||
"first_id": "eval_123",
|
||||
"last_id": "eval_123",
|
||||
"has_more": False,
|
||||
},
|
||||
request=httpx.Request("GET", "https://api.openai.com/v1/evals"),
|
||||
)
|
||||
|
||||
result = config.transform_list_evals_response(
|
||||
raw_response=response,
|
||||
logging_obj=None, # type: ignore
|
||||
)
|
||||
|
||||
assert result.object == "list"
|
||||
assert len(result.data) == 1
|
||||
assert result.data[0].id == "eval_123"
|
||||
assert result.has_more is False
|
||||
|
||||
|
||||
def test_transform_update_eval_request(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of update eval request"""
|
||||
update_request = {
|
||||
"name": "Updated Eval Name",
|
||||
"metadata": {"key": "value"},
|
||||
}
|
||||
|
||||
url, headers, request_body = config.transform_update_eval_request(
|
||||
eval_id="eval_123",
|
||||
update_request=update_request,
|
||||
api_base="https://api.openai.com",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123"
|
||||
assert request_body["name"] == "Updated Eval Name"
|
||||
assert request_body["metadata"]["key"] == "value"
|
||||
|
||||
|
||||
def test_transform_delete_eval_request(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of delete eval request"""
|
||||
url, headers = config.transform_delete_eval_request(
|
||||
eval_id="eval_123",
|
||||
api_base="https://api.openai.com",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123"
|
||||
|
||||
|
||||
def test_transform_delete_eval_response(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of delete eval response"""
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"object": "eval.deleted",
|
||||
"deleted": True,
|
||||
"eval_id": "eval_abc123"
|
||||
},
|
||||
request=httpx.Request("DELETE", "https://api.openai.com/v1/evals/eval_123"),
|
||||
)
|
||||
|
||||
result = config.transform_delete_eval_response(
|
||||
raw_response=response,
|
||||
logging_obj=None, # type: ignore
|
||||
)
|
||||
|
||||
assert result.eval_id == "eval_abc123"
|
||||
assert result.object == "eval.deleted"
|
||||
assert result.deleted is True
|
||||
|
||||
|
||||
def test_transform_cancel_eval_request(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of cancel eval request"""
|
||||
url, headers, request_body = config.transform_cancel_eval_request(
|
||||
eval_id="eval_123",
|
||||
api_base="https://api.openai.com",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == "https://api.openai.com/v1/evals/eval_123/cancel"
|
||||
assert request_body == {}
|
||||
|
||||
|
||||
def test_transform_cancel_eval_response(config: OpenAIEvalsConfig):
|
||||
"""Test transformation of cancel eval response"""
|
||||
response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "eval_123",
|
||||
"object": "eval",
|
||||
"status": "cancelled",
|
||||
},
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/evals/eval_123/cancel"),
|
||||
)
|
||||
|
||||
result = config.transform_cancel_eval_response(
|
||||
raw_response=response,
|
||||
logging_obj=None, # type: ignore
|
||||
)
|
||||
|
||||
assert result.id == "eval_123"
|
||||
assert result.object == "eval"
|
||||
Reference in New Issue
Block a user