[Feat] Containers API - add new container API file management + UI Interface (#17745)

* test_router_acreate_container_without_model

* _init_containers_api_endpoints

* test_init_containers_api_endpoints

* init container files endpoints

* init files api

* init container files API

* add containers api file content

* add code interpreter output UI

* add code interpreter input ui

* refactor code interpreter ui

* fix: require model selection

* cleaner container provision

* fix ContainerFileObject

* def container_file_content_handler(
add

* add retrieve_container_file_content

* aretrieve_container_file_content

* UI fix model

* fix linting errors
This commit is contained in:
Ishaan Jaff
2025-12-09 17:33:26 -08:00
committed by GitHub
parent 142567e143
commit 3631e8fa1d
22 changed files with 2716 additions and 21 deletions
+241
View File
@@ -0,0 +1,241 @@
# Container Files API
This module provides a unified interface for container file operations across multiple LLM providers (OpenAI, Azure OpenAI, etc.).
## Architecture
```
endpoints.json # Declarative endpoint definitions
endpoint_factory.py # Auto-generates SDK functions
container_handler.py # Generic HTTP handler
BaseContainerConfig # Provider-specific transformations
├── OpenAIContainerConfig
└── AzureContainerConfig (example)
```
## Files Overview
| File | Purpose |
|------|---------|
| `endpoints.json` | **Single source of truth** - Defines all container file endpoints |
| `endpoint_factory.py` | Auto-generates SDK functions (`list_container_files`, etc.) |
| `main.py` | Core container operations (create, list, retrieve, delete containers) |
| `utils.py` | Request parameter utilities |
## Adding a New Endpoint
To add a new container file endpoint (e.g., `get_container_file_content`):
### Step 1: Add to `endpoints.json`
```json
{
"name": "get_container_file_content",
"async_name": "aget_container_file_content",
"path": "/containers/{container_id}/files/{file_id}/content",
"method": "GET",
"path_params": ["container_id", "file_id"],
"query_params": [],
"response_type": "ContainerFileContentResponse"
}
```
### Step 2: Add Response Type (if new)
In `litellm/types/containers/main.py`:
```python
class ContainerFileContentResponse(BaseModel):
"""Response for file content download."""
content: bytes
# ... other fields
```
### Step 3: Register Response Type
In `litellm/llms/custom_httpx/container_handler.py`, add to `RESPONSE_TYPES`:
```python
RESPONSE_TYPES = {
# ... existing types
"ContainerFileContentResponse": ContainerFileContentResponse,
}
```
### Step 4: Update Router (one-time setup)
In `litellm/router.py`, add the call_type to the factory_function Literal and `_init_containers_api_endpoints` condition.
In `litellm/proxy/route_llm_request.py`, add to the route mappings and skip-model-routing lists.
### Step 5: Update Proxy Handler Factory (if new path params)
If your endpoint has a new combination of path parameters, add a handler in `litellm/proxy/container_endpoints/handler_factory.py`:
```python
elif path_params == ["container_id", "file_id", "new_param"]:
async def handler(...):
# handler implementation
```
---
## Adding a New Provider (e.g., Azure OpenAI)
### Step 1: Create Provider Config
Create `litellm/llms/azure/containers/transformation.py`:
```python
from typing import Dict, Optional, Tuple, Any
import httpx
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerFileObject,
DeleteContainerFileResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.secret_managers.main import get_secret_str
class AzureContainerConfig(BaseContainerConfig):
"""Configuration class for Azure OpenAI container API."""
def get_supported_openai_params(self) -> list:
return ["name", "expires_after", "file_ids", "extra_headers"]
def map_openai_params(
self,
container_create_optional_params,
drop_params: bool,
) -> Dict:
return dict(container_create_optional_params)
def validate_environment(
self,
headers: dict,
api_key: Optional[str] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm
api_key = (
api_key
or litellm.azure_key
or get_secret_str("AZURE_API_KEY")
)
headers["api-key"] = api_key
return headers
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Azure format:
https://{resource}.openai.azure.com/openai/containers?api-version=2024-xx
"""
if api_base is None:
raise ValueError("api_base is required for Azure")
api_version = litellm_params.get("api_version", "2024-02-15-preview")
return f"{api_base.rstrip('/')}/openai/containers?api-version={api_version}"
# Implement remaining abstract methods from BaseContainerConfig:
# - transform_container_create_request
# - transform_container_create_response
# - transform_container_list_request
# - transform_container_list_response
# - transform_container_retrieve_request
# - transform_container_retrieve_response
# - transform_container_delete_request
# - transform_container_delete_response
# - transform_container_file_list_request
# - transform_container_file_list_response
```
### Step 2: Register Provider Config
In `litellm/utils.py`, find `ProviderConfigManager.get_provider_container_config()` and add:
```python
@staticmethod
def get_provider_container_config(
provider: LlmProviders,
) -> Optional[BaseContainerConfig]:
if provider == LlmProviders.OPENAI:
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
return OpenAIContainerConfig()
elif provider == LlmProviders.AZURE:
from litellm.llms.azure.containers.transformation import AzureContainerConfig
return AzureContainerConfig()
return None
```
### Step 3: Test the New Provider
```bash
# Create container via Azure
curl -X POST "http://localhost:4000/v1/containers" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: azure" \
-H "Content-Type: application/json" \
-d '{"name": "My Azure Container"}'
# List container files via Azure
curl -X GET "http://localhost:4000/v1/containers/cntr_123/files" \
-H "Authorization: Bearer sk-1234" \
-H "custom-llm-provider: azure"
```
---
## How Provider Selection Works
1. **Proxy receives request** with `custom-llm-provider` header/query/body
2. **Router calls** `ProviderConfigManager.get_provider_container_config(provider)`
3. **Generic handler** uses the provider config for:
- URL construction (`get_complete_url`)
- Authentication (`validate_environment`)
- Request/response transformation
---
## Testing
Run the container API tests:
```bash
cd /Users/ishaanjaffer/github/litellm
python -m pytest tests/test_litellm/containers/ -v
```
Test via proxy:
```bash
# Start proxy
cd litellm/proxy && python proxy_cli.py --config proxy_config.yaml --port 4000
# Test endpoints
curl -X GET "http://localhost:4000/v1/containers/cntr_123/files" \
-H "Authorization: Bearer sk-1234"
```
---
## Endpoint Reference
| Endpoint | Method | Path |
|----------|--------|------|
| List container files | GET | `/v1/containers/{container_id}/files` |
| Retrieve container file | GET | `/v1/containers/{container_id}/files/{file_id}` |
| Delete container file | DELETE | `/v1/containers/{container_id}/files/{file_id}` |
See `endpoints.json` for the complete list.
+21
View File
@@ -1,5 +1,16 @@
"""Container management functions for LiteLLM."""
# Auto-generated container file functions from endpoints.json
from .endpoint_factory import (
adelete_container_file,
alist_container_files,
aretrieve_container_file,
aretrieve_container_file_content,
delete_container_file,
list_container_files,
retrieve_container_file,
retrieve_container_file_content,
)
from .main import (
acreate_container,
adelete_container,
@@ -12,6 +23,7 @@ from .main import (
)
__all__ = [
# Core container operations
"acreate_container",
"adelete_container",
"alist_containers",
@@ -20,5 +32,14 @@ __all__ = [
"delete_container",
"list_containers",
"retrieve_container",
# Container file operations (auto-generated from endpoints.json)
"adelete_container_file",
"alist_container_files",
"aretrieve_container_file",
"aretrieve_container_file_content",
"delete_container_file",
"list_container_files",
"retrieve_container_file",
"retrieve_container_file_content",
]
+224
View File
@@ -0,0 +1,224 @@
"""
Factory for generating container SDK functions from JSON config.
This module reads endpoints.json and dynamically generates SDK functions
that use the generic container handler.
"""
import asyncio
import contextvars
import json
from functools import partial
from pathlib import Path
from typing import Any, Callable, Dict, List, Literal, Optional, Type
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.llms.custom_httpx.container_handler import generic_container_handler
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerFileObject,
DeleteContainerFileResponse,
)
from litellm.types.router import GenericLiteLLMParams
from litellm.utils import ProviderConfigManager, client
# Response type mapping
RESPONSE_TYPES: Dict[str, Type] = {
"ContainerFileListResponse": ContainerFileListResponse,
"ContainerFileObject": ContainerFileObject,
"DeleteContainerFileResponse": DeleteContainerFileResponse,
}
def _load_endpoints_config() -> Dict:
"""Load the endpoints configuration from JSON file."""
config_path = Path(__file__).parent / "endpoints.json"
with open(config_path) as f:
return json.load(f)
def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
"""
Create a sync SDK function from endpoint config.
Uses the generic container handler instead of individual handler methods.
"""
endpoint_name = endpoint_config["name"]
response_type = RESPONSE_TYPES.get(endpoint_config["response_type"])
path_params = endpoint_config.get("path_params", [])
@client
def endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
):
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj")
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
# Check for mock response
mock_response = kwargs.get("mock_response")
if mock_response is not None:
if isinstance(mock_response, str):
mock_response = json.loads(mock_response)
if response_type:
return response_type(**mock_response)
return mock_response
# Get provider config
litellm_params = GenericLiteLLMParams(**kwargs)
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
raise ValueError(f"Container provider config not found for: {custom_llm_provider}")
# Build optional params for logging
optional_params = {k: kwargs.get(k) for k in path_params if k in kwargs}
# Pre-call logging
litellm_logging_obj.update_environment_variables(
model="",
optional_params=optional_params,
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=custom_llm_provider,
)
# Use generic handler
return generic_container_handler.handle(
endpoint_name=endpoint_name,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
**kwargs,
)
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
return endpoint_func
def create_async_endpoint_function(
sync_func: Callable,
endpoint_config: Dict,
) -> Callable:
"""Create an async SDK function that wraps the sync function."""
@client
async def async_endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
):
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["async_call"] = True
func = partial(
sync_func,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
return async_endpoint_func
def generate_container_endpoints() -> Dict[str, Callable]:
"""
Generate all container endpoint functions from the JSON config.
Returns a dict mapping function names to their implementations.
"""
config = _load_endpoints_config()
endpoints = {}
for endpoint_config in config["endpoints"]:
# Create sync function
sync_func = create_sync_endpoint_function(endpoint_config)
endpoints[endpoint_config["name"]] = sync_func
# Create async function
async_func = create_async_endpoint_function(sync_func, endpoint_config)
endpoints[endpoint_config["async_name"]] = async_func
return endpoints
def get_all_endpoint_names() -> List[str]:
"""Get all endpoint names (sync and async) from config."""
config = _load_endpoints_config()
names = []
for endpoint in config["endpoints"]:
names.append(endpoint["name"])
names.append(endpoint["async_name"])
return names
def get_async_endpoint_names() -> List[str]:
"""Get all async endpoint names for router registration."""
config = _load_endpoints_config()
return [endpoint["async_name"] for endpoint in config["endpoints"]]
# Generate endpoints on module load
_generated_endpoints = generate_container_endpoints()
# Export generated functions dynamically
list_container_files = _generated_endpoints.get("list_container_files")
alist_container_files = _generated_endpoints.get("alist_container_files")
retrieve_container_file = _generated_endpoints.get("retrieve_container_file")
aretrieve_container_file = _generated_endpoints.get("aretrieve_container_file")
delete_container_file = _generated_endpoints.get("delete_container_file")
adelete_container_file = _generated_endpoints.get("adelete_container_file")
retrieve_container_file_content = _generated_endpoints.get("retrieve_container_file_content")
aretrieve_container_file_content = _generated_endpoints.get("aretrieve_container_file_content")
+41
View File
@@ -0,0 +1,41 @@
{
"endpoints": [
{
"name": "list_container_files",
"async_name": "alist_container_files",
"path": "/containers/{container_id}/files",
"method": "GET",
"path_params": ["container_id"],
"query_params": ["after", "limit", "order"],
"response_type": "ContainerFileListResponse"
},
{
"name": "retrieve_container_file",
"async_name": "aretrieve_container_file",
"path": "/containers/{container_id}/files/{file_id}",
"method": "GET",
"path_params": ["container_id", "file_id"],
"query_params": [],
"response_type": "ContainerFileObject"
},
{
"name": "delete_container_file",
"async_name": "adelete_container_file",
"path": "/containers/{container_id}/files/{file_id}",
"method": "DELETE",
"path_params": ["container_id", "file_id"],
"query_params": [],
"response_type": "DeleteContainerFileResponse"
},
{
"name": "retrieve_container_file_content",
"async_name": "aretrieve_container_file_content",
"path": "/containers/{container_id}/files/{file_id}/content",
"method": "GET",
"path_params": ["container_id", "file_id"],
"query_params": [],
"response_type": "raw",
"returns_binary": true
}
]
}
+212
View File
@@ -12,6 +12,7 @@ from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.main import base_llm_http_handler
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerFileListResponse,
ContainerListOptionalRequestParams,
ContainerListResponse,
ContainerObject,
@@ -24,10 +25,12 @@ from litellm.utils import ProviderConfigManager, client
__all__ = [
"acreate_container",
"adelete_container",
"alist_container_files",
"alist_containers",
"aretrieve_container",
"create_container",
"delete_container",
"list_container_files",
"list_containers",
"retrieve_container",
]
@@ -147,6 +150,9 @@ def create_container(
expires_after: Optional[Dict[str, Any]] = None,
file_ids: Optional[List[str]] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -362,6 +368,9 @@ def list_containers(
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -547,6 +556,9 @@ def retrieve_container(
def retrieve_container(
container_id: str,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -724,6 +736,9 @@ def delete_container(
def delete_container(
container_id: str,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
@@ -799,3 +814,200 @@ def delete_container(
extra_kwargs=kwargs,
)
##### Container Files List #######################
@client
async def alist_container_files(
container_id: str,
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> ContainerFileListResponse:
"""Asynchronously list files in a container.
Parameters:
- `container_id` (str): The ID of the container
- `after` (Optional[str]): A cursor for pagination
- `limit` (Optional[int]): Number of items to return (1-100, default 20)
- `order` (Optional[str]): Sort order ('asc' or 'desc', default 'desc')
- `timeout` (int): Request timeout in seconds
- `custom_llm_provider` (Literal["openai"]): The LLM provider to use
- `extra_headers` (Optional[Dict[str, Any]]): Additional headers
- `extra_query` (Optional[Dict[str, Any]]): Additional query parameters
- `extra_body` (Optional[Dict[str, Any]]): Additional body parameters
- `kwargs` (dict): Additional keyword arguments
Returns:
- `response` (ContainerFileListResponse): The list of container files
"""
local_vars = locals()
try:
loop = asyncio.get_event_loop()
kwargs["async_call"] = True
func = partial(
list_container_files,
container_id=container_id,
after=after,
limit=limit,
order=order,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
**kwargs,
)
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
init_response = await loop.run_in_executor(None, func_with_context)
if asyncio.iscoroutine(init_response):
response = await init_response
else:
response = init_response
return response
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
# fmt: off
@overload
def list_container_files(
container_id: str,
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
*,
alist_container_files: Literal[True],
**kwargs,
) -> Coroutine[Any, Any, ContainerFileListResponse]:
...
@overload
def list_container_files(
container_id: str,
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
*,
alist_container_files: Literal[False] = False,
**kwargs,
) -> ContainerFileListResponse:
...
# fmt: on
@client
def list_container_files(
container_id: str,
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[
ContainerFileListResponse,
Coroutine[Any, Any, ContainerFileListResponse],
]:
"""List files in a container using the OpenAI Container API.
Currently supports OpenAI
"""
local_vars = locals()
try:
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
# Check for mock response first
mock_response = kwargs.get("mock_response")
if mock_response is not None:
if isinstance(mock_response, str):
mock_response = json.loads(mock_response)
response = ContainerFileListResponse(**mock_response)
return response
# get llm provider logic
litellm_params = GenericLiteLLMParams(**kwargs)
# get provider config
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
raise ValueError(f"Container provider config not found for provider: {custom_llm_provider}")
# Pre Call logging
litellm_logging_obj.update_environment_variables(
model="",
optional_params={"container_id": container_id, "after": after, "limit": limit, "order": order},
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.list_container_files.value
return base_llm_http_handler.container_file_list_handler(
container_id=container_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
after=after,
limit=limit,
order=order,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
)
@@ -12,11 +12,12 @@ from litellm.types.router import GenericLiteLLMParams
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.types.containers.main import (
ContainerListResponse as _ContainerListResponse,
ContainerFileListResponse as _ContainerFileListResponse,
)
from litellm.types.containers.main import (
ContainerObject as _ContainerObject,
ContainerListResponse as _ContainerListResponse,
)
from litellm.types.containers.main import ContainerObject as _ContainerObject
from litellm.types.containers.main import (
DeleteContainerResult as _DeleteContainerResult,
)
@@ -28,12 +29,14 @@ if TYPE_CHECKING:
ContainerObject = _ContainerObject
DeleteContainerResult = _DeleteContainerResult
ContainerListResponse = _ContainerListResponse
ContainerFileListResponse = _ContainerFileListResponse
else:
LiteLLMLoggingObj = Any
BaseLLMException = Any
ContainerObject = Any
DeleteContainerResult = Any
ContainerListResponse = Any
ContainerFileListResponse = Any
class BaseContainerConfig(ABC):
@@ -193,6 +196,63 @@ class BaseContainerConfig(ABC):
"""Transform the container delete response."""
...
@abstractmethod
def transform_container_file_list_request(
self,
container_id: str,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
after: str | None = None,
limit: int | None = None,
order: str | None = None,
extra_query: dict[str, Any] | None = None,
) -> tuple[str, dict]:
"""Transform the container file list request into a URL and params.
Returns:
tuple[str, dict]: (url, params) for the container file list request.
"""
...
@abstractmethod
def transform_container_file_list_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ContainerFileListResponse:
"""Transform the container file list response."""
...
@abstractmethod
def transform_container_file_content_request(
self,
container_id: str,
file_id: str,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[str, dict]:
"""Transform the container file content request into a URL and params.
Returns:
tuple[str, dict]: (url, params) for the container file content request.
"""
...
@abstractmethod
def transform_container_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> bytes:
"""Transform the container file content response.
Returns:
bytes: The raw file content.
"""
...
def get_error_class(
self,
error_message: str,
@@ -0,0 +1,348 @@
"""
Generic container file handler for LiteLLM.
This module provides a single generic handler that can process any container file
endpoint defined in endpoints.json, eliminating the need for individual handler methods.
"""
import json
from pathlib import Path
from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union
import httpx
import litellm
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
_get_httpx_client,
get_async_httpx_client,
)
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerFileObject,
DeleteContainerFileResponse,
)
from litellm.types.router import GenericLiteLLMParams
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
# Response type mapping
RESPONSE_TYPES: Dict[str, Type] = {
"ContainerFileListResponse": ContainerFileListResponse,
"ContainerFileObject": ContainerFileObject,
"DeleteContainerFileResponse": DeleteContainerFileResponse,
}
def _load_endpoints_config() -> Dict:
"""Load the endpoints configuration from JSON file."""
config_path = Path(__file__).parent.parent.parent / "containers" / "endpoints.json"
with open(config_path) as f:
return json.load(f)
def _get_endpoint_config(endpoint_name: str) -> Optional[Dict]:
"""Get config for a specific endpoint by name."""
config = _load_endpoints_config()
for endpoint in config["endpoints"]:
if endpoint["name"] == endpoint_name or endpoint["async_name"] == endpoint_name:
return endpoint
return None
def _build_url(
api_base: str,
path_template: str,
path_params: Dict[str, str],
) -> str:
"""Build the full URL by substituting path parameters.
The api_base from get_complete_url already includes /containers,
so we need to strip that prefix from the path_template.
"""
# api_base ends with /containers, path_template starts with /containers
# So we need to strip /containers from the path
if path_template.startswith("/containers"):
path_template = path_template[len("/containers"):]
url = f"{api_base.rstrip('/')}{path_template}"
for param, value in path_params.items():
url = url.replace(f"{{{param}}}", value)
return url
def _build_query_params(
query_param_names: list,
kwargs: Dict[str, Any],
) -> Dict[str, str]:
"""Build query parameters from kwargs."""
params = {}
for param_name in query_param_names:
value = kwargs.get(param_name)
if value is not None:
params[param_name] = str(value) if not isinstance(value, str) else value
return params
class GenericContainerHandler:
"""
Generic handler for container file API endpoints.
This single handler can process any endpoint defined in endpoints.json,
eliminating the need for individual handler methods per endpoint.
"""
def handle(
self,
endpoint_name: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
**kwargs,
) -> Union[Any, Coroutine[Any, Any, Any]]:
"""
Generic handler for any container file endpoint.
Args:
endpoint_name: Name of the endpoint (e.g., "list_container_files")
container_provider_config: Provider-specific configuration
litellm_params: LiteLLM parameters including api_key, api_base
logging_obj: Logging object for request logging
extra_headers: Additional HTTP headers
extra_query: Additional query parameters
timeout: Request timeout
_is_async: Whether to make async request
client: Optional HTTP client
**kwargs: Path params and query params (e.g., container_id, file_id, after, limit)
"""
if _is_async:
return self._async_handle(
endpoint_name=endpoint_name,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout,
client=client,
**kwargs,
)
return self._sync_handle(
endpoint_name=endpoint_name,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout,
client=client,
**kwargs,
)
def _sync_handle(
self,
endpoint_name: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
**kwargs,
) -> Any:
"""Synchronous request handler."""
endpoint_config = _get_endpoint_config(endpoint_name)
if not endpoint_config:
raise ValueError(f"Unknown endpoint: {endpoint_name}")
# Get HTTP client
if client is None or not isinstance(client, HTTPHandler):
http_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
)
else:
http_client = client
# Build request
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Build URL with path params
path_params = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])}
url = _build_url(api_base, endpoint_config["path"], path_params)
# Build query params
query_params = _build_query_params(endpoint_config.get("query_params", []), kwargs)
if extra_query:
query_params.update(extra_query)
# Log request
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": query_params,
},
)
# Make request
method = endpoint_config["method"].upper()
returns_binary = endpoint_config.get("returns_binary", False)
try:
if method == "GET":
response = http_client.get(url=url, headers=headers, params=query_params)
elif method == "DELETE":
response = http_client.delete(url=url, headers=headers, params=query_params)
elif method == "POST":
response = http_client.post(url=url, headers=headers, params=query_params)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
# For binary responses, return raw content
if returns_binary:
return response.content
# Check for error response
response_json = response.json()
if "error" in response_json:
from litellm.llms.base_llm.chat.transformation import BaseLLMException
error_msg = response_json.get("error", {}).get("message", str(response_json))
raise BaseLLMException(
status_code=response.status_code,
message=error_msg,
headers=dict(response.headers),
)
# Parse response
response_type = RESPONSE_TYPES.get(endpoint_config["response_type"])
if response_type:
return response_type(**response_json)
return response_json
except Exception as e:
raise e
async def _async_handle(
self,
endpoint_name: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
**kwargs,
) -> Any:
"""Asynchronous request handler."""
endpoint_config = _get_endpoint_config(endpoint_name)
if not endpoint_config:
raise ValueError(f"Unknown endpoint: {endpoint_name}")
# Get HTTP client
if client is None or not isinstance(client, AsyncHTTPHandler):
http_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
http_client = client
# Build request
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Build URL with path params
path_params = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])}
url = _build_url(api_base, endpoint_config["path"], path_params)
# Build query params
query_params = _build_query_params(endpoint_config.get("query_params", []), kwargs)
if extra_query:
query_params.update(extra_query)
# Log request
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": query_params,
},
)
# Make request
method = endpoint_config["method"].upper()
returns_binary = endpoint_config.get("returns_binary", False)
try:
if method == "GET":
response = await http_client.get(url=url, headers=headers, params=query_params)
elif method == "DELETE":
response = await http_client.delete(url=url, headers=headers, params=query_params)
elif method == "POST":
response = await http_client.post(url=url, headers=headers, params=query_params)
else:
raise ValueError(f"Unsupported HTTP method: {method}")
# For binary responses, return raw content
if returns_binary:
return response.content
# Check for error response
response_json = response.json()
if "error" in response_json:
from litellm.llms.base_llm.chat.transformation import BaseLLMException
error_msg = response_json.get("error", {}).get("message", str(response_json))
raise BaseLLMException(
status_code=response.status_code,
message=error_msg,
headers=dict(response.headers),
)
# Parse response
response_type = RESPONSE_TYPES.get(endpoint_config["response_type"])
if response_type:
return response_type(**response_json)
return response_json
except Exception as e:
raise e
# Singleton instance
generic_container_handler = GenericContainerHandler()
@@ -66,6 +66,7 @@ from litellm.responses.streaming_iterator import (
SyncResponsesAPIStreamingIterator,
)
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
@@ -5711,6 +5712,337 @@ class BaseLLMHTTPHandler:
provider_config=container_provider_config,
)
def container_file_list_handler(
self,
container_id: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
) -> Union["ContainerFileListResponse", Coroutine[Any, Any, "ContainerFileListResponse"]]:
if _is_async:
return self.async_container_file_list_handler(
container_id=container_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=logging_obj,
after=after,
limit=limit,
order=order,
extra_headers=extra_headers,
extra_query=extra_query,
timeout=timeout,
client=client,
)
# For sync calls, use sync HTTP client
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
)
else:
sync_httpx_client = client
# Validate environment and get headers
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
# Get the complete URL for container files
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Transform the request using the provider config
url, params = container_provider_config.transform_container_file_list_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
after=after,
limit=limit,
order=order,
extra_query=extra_query,
)
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": params,
},
)
try:
response = sync_httpx_client.get(
url=url,
headers=headers,
params=params,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
async def async_container_file_list_handler(
self,
container_id: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
) -> "ContainerFileListResponse":
# For async calls, use async HTTP client
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
async_httpx_client = client
# Validate environment and get headers
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
# Get the complete URL for container files
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Transform the request using the provider config
url, params = container_provider_config.transform_container_file_list_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
after=after,
limit=limit,
order=order,
extra_query=extra_query,
)
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": params,
},
)
try:
response = await async_httpx_client.get(
url=url,
headers=headers,
params=params,
)
return container_provider_config.transform_container_file_list_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
def container_file_content_handler(
self,
container_id: str,
file_id: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
_is_async: bool = False,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
) -> Union[bytes, Coroutine[Any, Any, bytes]]:
if _is_async:
return self.async_container_file_content_handler(
container_id=container_id,
file_id=file_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
timeout=timeout,
client=client,
)
# For sync calls, use sync HTTP client
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
)
else:
sync_httpx_client = client
# Validate environment and get headers
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
# Get the complete URL for container files
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Transform the request using the provider config
url, params = container_provider_config.transform_container_file_content_request(
container_id=container_id,
file_id=file_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": params,
},
)
try:
response = sync_httpx_client.get(
url=url,
headers=headers,
params=params,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
async def async_container_file_content_handler(
self,
container_id: str,
file_id: str,
container_provider_config: "BaseContainerConfig",
litellm_params: GenericLiteLLMParams,
logging_obj: "LiteLLMLoggingObj",
extra_headers: Optional[Dict[str, Any]] = None,
timeout: Union[float, httpx.Timeout] = 600,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
) -> bytes:
# For async calls, use async HTTP client
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
async_httpx_client = client
# Validate environment and get headers
headers = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
)
if extra_headers:
headers.update(extra_headers)
# Get the complete URL for container files
api_base = container_provider_config.get_complete_url(
api_base=litellm_params.get("api_base", None),
litellm_params=dict(litellm_params),
)
# Transform the request using the provider config
url, params = container_provider_config.transform_container_file_content_request(
container_id=container_id,
file_id=file_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
## LOGGING
logging_obj.pre_call(
input="",
api_key="",
additional_args={
"api_base": url,
"headers": headers,
"params": params,
},
)
try:
response = await async_httpx_client.get(
url=url,
headers=headers,
params=params,
)
return container_provider_config.transform_container_file_content_response(
raw_response=response,
logging_obj=logging_obj,
)
except Exception as e:
raise self._handle_error(
e=e,
provider_config=container_provider_config,
)
###### VECTOR STORE HANDLER ######
async def async_vector_store_search_handler(
self,
@@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
from litellm.secret_managers.main import get_secret_str
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerFileListResponse,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
@@ -19,7 +20,9 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException
from ...base_llm.containers.transformation import BaseContainerConfig as _BaseContainerConfig
from ...base_llm.containers.transformation import (
BaseContainerConfig as _BaseContainerConfig,
)
LiteLLMLoggingObj = _LiteLLMLoggingObj
BaseContainerConfig = _BaseContainerConfig
@@ -247,6 +250,86 @@ class OpenAIContainerConfig(BaseContainerConfig):
return delete_result
def transform_container_file_list_request(
self,
container_id: str,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
after: Optional[str] = None,
limit: Optional[int] = None,
order: Optional[str] = None,
extra_query: Optional[Dict[str, Any]] = None,
) -> Tuple[str, Dict]:
"""Transform the container file list request for OpenAI API.
OpenAI API expects the following request:
- GET /v1/containers/{container_id}/files
"""
# Construct the URL for container files
url = f"{api_base.rstrip('/')}/{container_id}/files"
# Prepare query parameters
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
if limit is not None:
params["limit"] = str(limit)
if order is not None:
params["order"] = order
# Add any extra query parameters
if extra_query:
params.update(extra_query)
return url, params
def transform_container_file_list_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> ContainerFileListResponse:
"""Transform the OpenAI container file list response.
"""
response_data = raw_response.json()
# Transform the response data
file_list = ContainerFileListResponse(**response_data) # type: ignore[arg-type]
return file_list
def transform_container_file_content_request(
self,
container_id: str,
file_id: str,
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
"""Transform the container file content request for OpenAI API.
OpenAI API expects the following request:
- GET /v1/containers/{container_id}/files/{file_id}/content
"""
# Construct the URL for container file content
url = f"{api_base.rstrip('/')}/{container_id}/files/{file_id}/content"
# No query parameters needed
params: Dict[str, Any] = {}
return url, params
def transform_container_file_content_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> bytes:
"""Transform the OpenAI container file content response.
Returns the raw binary content of the file.
"""
return raw_response.content
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers],
) -> BaseLLMException:
@@ -1,6 +1,7 @@
#### Container Endpoints #####
from typing import Any, Dict
from fastapi import APIRouter, Depends, Request, Response
from fastapi.responses import ORJSONResponse
@@ -9,9 +10,9 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_au
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_body,
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
get_custom_llm_provider_from_request_body,
)
router = APIRouter()
@@ -404,3 +405,10 @@ async def delete_container(
version=version,
)
# Register JSON-configured container file endpoints
from litellm.proxy.container_endpoints.handler_factory import (
register_container_file_endpoints,
)
register_container_file_endpoints(router)
@@ -0,0 +1,310 @@
"""
Factory for generating container proxy endpoints from JSON config.
This module reads the endpoints.json config and dynamically creates
FastAPI route handlers for ALL container file endpoints.
"""
import json
from pathlib import Path
from typing import Any, Dict, List
from fastapi import APIRouter, Depends, Request, Response
from fastapi.responses import ORJSONResponse
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
)
def _load_endpoints_config() -> Dict:
"""Load the endpoints configuration from JSON file."""
config_path = Path(__file__).parent.parent.parent / "containers" / "endpoints.json"
with open(config_path) as f:
return json.load(f)
def get_all_route_types() -> List[str]:
"""Get all async route types for registration in route_llm_request.py"""
config = _load_endpoints_config()
return [endpoint["async_name"] for endpoint in config["endpoints"]]
def _get_container_provider_config(custom_llm_provider: str):
"""Get the container provider config for the given provider."""
if custom_llm_provider == "openai":
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
return OpenAIContainerConfig()
else:
raise ValueError(f"Container API not supported for provider: {custom_llm_provider}")
def _create_handler_for_path_params(path_params: List[str], route_type: str, returns_binary: bool = False):
"""
Dynamically create a handler with the correct path parameter signature.
"""
# For binary content endpoints, use a different handler
if returns_binary and path_params == ["container_id", "file_id"]:
async def handler_binary_content(
request: Request,
container_id: str,
file_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
return await _process_binary_request(
request=request,
container_id=container_id,
file_id=file_id,
user_api_key_dict=user_api_key_dict,
)
return handler_binary_content
# Create handlers for different path parameter combinations
if path_params == ["container_id"]:
async def handler_container_id(
request: Request,
container_id: str,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
return await _process_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={"container_id": container_id},
)
return handler_container_id
elif path_params == ["container_id", "file_id"]:
async def handler_container_file(
request: Request,
container_id: str,
file_id: str,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
return await _process_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={"container_id": container_id, "file_id": file_id},
)
return handler_container_file
else:
# Fallback for no path params
async def handler_no_params(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
return await _process_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type=route_type,
path_params={},
)
return handler_no_params
async def _process_binary_request(
request: Request,
container_id: str,
file_id: str,
user_api_key_dict: UserAPIKeyAuth,
):
"""
Process binary content requests using the proper transformation pattern.
This uses the provider config transformations and llm_http_handler
to maintain consistency with the established pattern.
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.types.router import GenericLiteLLMParams
# Extract custom_llm_provider
custom_llm_provider = (
get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
# Get the provider config
container_provider_config = _get_container_provider_config(custom_llm_provider)
# Build litellm_params - credentials are resolved by provider config from env
litellm_params = GenericLiteLLMParams()
# Create logging object
logging_obj = Logging(
model="container-file-content",
messages=[],
stream=False,
call_type="container_file_content",
start_time=None,
litellm_call_id="",
function_id="",
)
# Use the HTTP handler to make the request
handler = BaseLLMHTTPHandler()
try:
content = await handler.async_container_file_content_handler(
container_id=container_id,
file_id=file_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=logging_obj,
)
# Determine content type based on common file extensions in the file_id
content_type = "application/octet-stream"
file_id_lower = file_id.lower()
if ".png" in file_id_lower or file_id_lower.endswith("png"):
content_type = "image/png"
elif ".jpg" in file_id_lower or ".jpeg" in file_id_lower:
content_type = "image/jpeg"
elif ".gif" in file_id_lower:
content_type = "image/gif"
elif ".csv" in file_id_lower:
content_type = "text/csv"
elif ".json" in file_id_lower:
content_type = "application/json"
elif ".txt" in file_id_lower:
content_type = "text/plain"
elif ".pdf" in file_id_lower:
content_type = "application/pdf"
return Response(
content=content,
media_type=content_type,
)
except Exception as e:
raise e
async def _process_request(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth,
route_type: str,
path_params: Dict[str, str],
):
"""Common request processing logic."""
from litellm.proxy.proxy_server import (
general_settings,
llm_router,
proxy_config,
proxy_logging_obj,
select_data_generator,
user_api_base,
user_max_tokens,
user_model,
user_request_timeout,
user_temperature,
version,
)
query_params = dict(request.query_params)
data: Dict[str, Any] = {
"query_params": query_params,
**path_params,
}
custom_llm_provider = (
get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
data["custom_llm_provider"] = custom_llm_provider
processor = ProxyBaseLLMRequestProcessing(data=data)
try:
return await processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
user_api_key_dict=user_api_key_dict,
route_type=route_type, # type: ignore[arg-type]
proxy_logging_obj=proxy_logging_obj,
llm_router=llm_router,
general_settings=general_settings,
proxy_config=proxy_config,
select_data_generator=select_data_generator,
model=None,
user_model=user_model,
user_temperature=user_temperature,
user_request_timeout=user_request_timeout,
user_max_tokens=user_max_tokens,
user_api_base=user_api_base,
version=version,
)
except Exception as e:
raise await processor._handle_llm_api_exception(
e=e,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
version=version,
)
def register_container_file_endpoints(router: APIRouter) -> None:
"""
Register ALL container file endpoints from JSON config to the router.
This single function registers all endpoints defined in endpoints.json,
eliminating the need for manual endpoint definitions.
"""
config = _load_endpoints_config()
for endpoint_config in config["endpoints"]:
path = endpoint_config["path"]
method = endpoint_config["method"].lower()
path_params = endpoint_config.get("path_params", [])
route_type = endpoint_config["async_name"]
returns_binary = endpoint_config.get("returns_binary", False)
# Create handler with correct signature for path params
handler = _create_handler_for_path_params(path_params, route_type, returns_binary)
# Register routes
route_method = getattr(router, method)
# For binary endpoints, don't use ORJSONResponse
if returns_binary:
# Register both /v1/... and /... paths without JSON response class
route_method(
f"/v1{path}",
dependencies=[Depends(user_api_key_auth)],
tags=["containers"],
)(handler)
route_method(
path,
dependencies=[Depends(user_api_key_auth)],
tags=["containers"],
)(handler)
else:
# Register both /v1/... and /... paths with JSON response
route_method(
f"/v1{path}",
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse,
tags=["containers"],
)(handler)
route_method(
path,
dependencies=[Depends(user_api_key_auth)],
response_class=ORJSONResponse,
tags=["containers"],
)(handler)
+17
View File
@@ -36,6 +36,11 @@ ROUTE_ENDPOINT_MAPPING = {
"alist_containers": "/containers",
"aretrieve_container": "/containers/{container_id}",
"adelete_container": "/containers/{container_id}",
# Auto-generated container file routes
"alist_container_files": "/containers/{container_id}/files",
"aretrieve_container_file": "/containers/{container_id}/files/{file_id}",
"adelete_container_file": "/containers/{container_id}/files/{file_id}",
"aretrieve_container_file_content": "/containers/{container_id}/files/{file_id}/content",
"acreate_skill": "/skills",
"alist_skills": "/skills",
"aget_skill": "/skills/{skill_id}",
@@ -132,6 +137,10 @@ async def route_request(
"alist_containers",
"aretrieve_container",
"adelete_container",
"alist_container_files",
"aretrieve_container_file",
"adelete_container_file",
"aretrieve_container_file_content",
"acreate_skill",
"alist_skills",
"aget_skill",
@@ -184,6 +193,10 @@ async def route_request(
"alist_containers",
"aretrieve_container",
"adelete_container",
"alist_container_files",
"aretrieve_container_file",
"adelete_container_file",
"aretrieve_container_file_content",
]:
return getattr(llm_router, f"{route_type}")(**data)
if route_type in [
@@ -256,6 +269,10 @@ async def route_request(
"alist_containers",
"aretrieve_container",
"adelete_container",
"alist_container_files",
"aretrieve_container_file",
"adelete_container_file",
"aretrieve_container_file_content",
]:
# moderation endpoint does not require `model` parameter
return getattr(llm_router, f"{route_type}")(**data)
+17
View File
@@ -1017,6 +1017,9 @@ class Router:
list_containers,
retrieve_container,
)
from litellm.containers.endpoint_factory import (
_generated_endpoints as container_file_endpoints,
)
self.acreate_container = self.factory_function(
acreate_container, call_type="acreate_container"
@@ -1042,6 +1045,10 @@ class Router:
self.delete_container = self.factory_function(
delete_container, call_type="delete_container"
)
# Auto-register JSON-generated container file endpoints
for name, func in container_file_endpoints.items():
setattr(self, name, self.factory_function(func, call_type=name)) # type: ignore[arg-type]
def _initialize_skills_endpoints(self):
"""Initialize Anthropic Skills API endpoints."""
@@ -3841,6 +3848,12 @@ class Router:
"retrieve_container",
"adelete_container",
"delete_container",
"alist_container_files",
"list_container_files",
"aretrieve_container_file",
"retrieve_container_file",
"adelete_container_file",
"delete_container_file",
"acreate_skill",
"alist_skills",
"aget_skill",
@@ -3976,6 +3989,10 @@ class Router:
"alist_containers",
"aretrieve_container",
"adelete_container",
"alist_container_files",
"aretrieve_container_file",
"adelete_container_file",
"aretrieve_container_file_content",
):
return await self._init_containers_api_endpoints(
original_function=original_function,
+74 -1
View File
@@ -1,7 +1,7 @@
from typing import Any, Dict, List, Literal, Optional
from typing_extensions import TypedDict
from pydantic import BaseModel
from typing_extensions import TypedDict
class ExpiresAfter(BaseModel):
@@ -120,3 +120,76 @@ class ContainerListOptionalRequestParams(TypedDict, total=False):
extra_headers: Optional[Dict[str, str]]
extra_query: Optional[Dict[str, str]]
class ContainerFileObject(BaseModel):
"""Represents a container file object."""
id: str
object: Literal["container.file", "container_file"] # OpenAI returns "container.file"
container_id: str
bytes: Optional[int] = None # Can be null for some files
created_at: int
path: str
source: str
_hidden_params: Dict[str, Any] = {}
def __contains__(self, key):
return hasattr(self, key)
def get(self, key, default=None):
return getattr(self, key, default)
def __getitem__(self, key):
return getattr(self, key)
def json(self, **kwargs): # type: ignore
try:
return self.model_dump(**kwargs)
except Exception:
return self.dict()
class ContainerFileListResponse(BaseModel):
"""Response object for list container files request."""
object: Literal["list"]
data: List[ContainerFileObject]
first_id: Optional[str] = None
last_id: Optional[str] = None
has_more: bool
def __contains__(self, key):
return hasattr(self, key)
def get(self, key, default=None):
return getattr(self, key, default)
def __getitem__(self, key):
return getattr(self, key)
def json(self, **kwargs): # type: ignore
try:
return self.model_dump(**kwargs)
except Exception:
return self.dict()
class DeleteContainerFileResponse(BaseModel):
"""Response object for delete container file request."""
id: str
object: Literal["container_file.deleted"]
deleted: bool
def __contains__(self, key):
return hasattr(self, key)
def get(self, key, default=None):
return getattr(self, key, default)
def __getitem__(self, key):
return getattr(self, key)
def json(self, **kwargs): # type: ignore
try:
return self.model_dump(**kwargs)
except Exception:
return self.dict()
+2
View File
@@ -319,6 +319,8 @@ class CallTypes(str, Enum):
aretrieve_container = "aretrieve_container"
delete_container = "delete_container"
adelete_container = "adelete_container"
list_container_files = "list_container_files"
alist_container_files = "alist_container_files"
acancel_fine_tuning_job = "acancel_fine_tuning_job"
cancel_fine_tuning_job = "cancel_fine_tuning_job"
@@ -0,0 +1,119 @@
"""
E2E Test for Container Files API.
Tests the container files endpoints using LiteLLM SDK methods.
"""
import os
import sys
import time
import pytest
sys.path.insert(0, os.path.abspath("../.."))
from litellm.containers import (
create_container,
delete_container,
)
from litellm.containers.endpoint_factory import (
list_container_files,
retrieve_container_file,
retrieve_container_file_content,
delete_container_file,
)
@pytest.mark.skipif(
not os.getenv("OPENAI_API_KEY"),
reason="OPENAI_API_KEY not set"
)
def test_container_files_api():
"""
Test container files API: list, retrieve, delete.
Flow:
1. Create a container
2. List files (should be empty)
3. Try retrieve file (should error - no files)
4. Try delete file (should error - no files)
5. Cleanup: delete container
"""
api_key = os.getenv("OPENAI_API_KEY")
# 1. Create container
print("\n1. Creating container...")
container = create_container(
name=f"test-files-api-{int(time.time())}",
custom_llm_provider="openai",
api_key=api_key,
expires_after={"anchor": "last_active_at", "minutes": 5},
)
print(f" Created: {container.id}")
try:
# 2. List files
print("2. Listing container files...")
files = list_container_files(
container_id=container.id,
custom_llm_provider="openai",
api_key=api_key,
)
assert files.object == "list"
assert isinstance(files.data, list)
assert len(files.data) == 0 # New container has no files
print(f" Files found: {len(files.data)}")
# 3. Try retrieve non-existent file metadata (should raise error)
print("3. Testing retrieve_container_file (expect error)...")
try:
retrieve_container_file(
container_id=container.id,
file_id="cfile_nonexistent",
custom_llm_provider="openai",
api_key=api_key,
)
assert False, "Should have raised error for non-existent file"
except Exception as e:
assert "not found" in str(e).lower() or "invalid" in str(e).lower()
print(f" Got expected error ✓")
# 3b. Try retrieve non-existent file content (should raise error)
print("3b. Testing retrieve_container_file_content (expect error)...")
try:
retrieve_container_file_content(
container_id=container.id,
file_id="cfile_nonexistent",
custom_llm_provider="openai",
api_key=api_key,
)
assert False, "Should have raised error for non-existent file content"
except Exception as e:
print(f" Got expected error ✓")
# 4. Try delete non-existent file (should raise error)
print("4. Testing delete_container_file (expect error)...")
try:
delete_container_file(
container_id=container.id,
file_id="cfile_nonexistent",
custom_llm_provider="openai",
api_key=api_key,
)
assert False, "Should have raised error for non-existent file"
except Exception as e:
# Delete returns 400 for non-existent files
print(f" Got expected error ✓")
finally:
# 5. Cleanup
print("5. Deleting container...")
result = delete_container(
container_id=container.id,
custom_llm_provider="openai",
api_key=api_key,
)
assert result.deleted is True
print(f" Deleted ✓")
print("\nAll container files API tests passed! ✓")
@@ -51,6 +51,8 @@ import { fetchAvailableModels, ModelGroup } from "../llm_calls/fetch_models";
import { makeOpenAIImageEditsRequest } from "../llm_calls/image_edits";
import { makeOpenAIImageGenerationRequest } from "../llm_calls/image_generation";
import { makeOpenAIResponsesRequest } from "../llm_calls/responses_api";
import CodeInterpreterOutput from "./CodeInterpreterOutput";
import { useCodeInterpreter } from "./useCodeInterpreter";
import { Agent, fetchAvailableAgents } from "../llm_calls/fetch_agents";
import { makeA2AStreamMessageRequest } from "../llm_calls/a2a_send_message";
import A2AMetrics from "./A2AMetrics";
@@ -65,6 +67,7 @@ import { createDisplayMessage, createMultimodalMessage } from "./ResponsesImageU
import { SearchResultsDisplay } from "./SearchResultsDisplay";
import SessionManagement from "./SessionManagement";
import { MessageType } from "./types";
import CodeInterpreterTool from "./CodeInterpreterTool";
const { TextArea } = Input;
const { Dragger } = Upload;
@@ -197,6 +200,9 @@ const ChatUI: React.FC<ChatUIProps> = ({
const [temperature, setTemperature] = useState<number>(1.0);
const [maxTokens, setMaxTokens] = useState<number>(2048);
const [useAdvancedParams, setUseAdvancedParams] = useState<boolean>(false);
// Code Interpreter state (using custom hook)
const codeInterpreter = useCodeInterpreter();
const chatEndRef = useRef<HTMLDivElement>(null);
@@ -295,6 +301,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
sessionStorage.removeItem("responsesSessionId");
}
sessionStorage.setItem("useApiSessionManagement", JSON.stringify(useApiSessionManagement));
// Note: codeInterpreterEnabled and selectedContainerId are persisted by useCodeInterpreter hook
}, [
apiKeySource,
apiKey,
@@ -736,6 +743,12 @@ const ChatUI: React.FC<ChatUIProps> = ({
return;
}
// Require model selection for Responses API
if (endpointType === EndpointType.RESPONSES && !selectedModel) {
NotificationsManager.fromBackend("Please select a model before sending a request");
return;
}
if (!token || !userRole || !userID) {
return;
}
@@ -809,6 +822,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
setChatHistory([...chatHistory, displayMessage]);
setMCPEvents([]); // Clear previous MCP events for new conversation turn
codeInterpreter.clearResult(); // Clear previous code interpreter results
setIsLoading(true);
try {
@@ -914,6 +928,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
useApiSessionManagement ? responsesSessionId : null, // Only pass session ID if API mode is enabled
handleResponseId, // Pass callback to capture new response ID
handleMCPEvent, // Pass MCP event handler
codeInterpreter.enabled, // Enable Code Interpreter tool
codeInterpreter.setResult, // Handle code interpreter output
);
} else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) {
const apiChatHistory = [
@@ -1388,6 +1404,20 @@ const ChatUI: React.FC<ChatUIProps> = ({
accessToken={accessToken || ""}
/>
</div>
{/* Code Interpreter Toggle - Only for Responses endpoint */}
{endpointType === EndpointType.RESPONSES && (
<div>
<CodeInterpreterTool
accessToken={apiKeySource === "session" ? accessToken || "" : apiKey}
enabled={codeInterpreter.enabled}
onEnabledChange={codeInterpreter.setEnabled}
selectedContainerId={null}
onContainerChange={() => {}}
selectedModel={selectedModel || ""}
/>
</div>
)}
</div>
</div>
@@ -1468,6 +1498,19 @@ const ChatUI: React.FC<ChatUIProps> = ({
<SearchResultsDisplay searchResults={message.searchResults} />
)}
{/* Show Code Interpreter output for the last assistant message */}
{message.role === "assistant" &&
index === chatHistory.length - 1 &&
codeInterpreter.result &&
endpointType === EndpointType.RESPONSES && (
<CodeInterpreterOutput
code={codeInterpreter.result.code}
containerId={codeInterpreter.result.containerId}
annotations={codeInterpreter.result.annotations}
accessToken={apiKeySource === "session" ? accessToken || "" : apiKey}
/>
)}
<div
className="whitespace-pre-wrap break-words max-w-full message-content"
style={{
@@ -1772,10 +1815,55 @@ const ChatUI: React.FC<ChatUIProps> = ({
</div>
)}
{/* Code Interpreter indicator and sample prompts when enabled */}
{endpointType === EndpointType.RESPONSES && codeInterpreter.enabled && (
<div className="mb-2 space-y-2">
<div className="px-3 py-2 bg-gradient-to-r from-blue-50 to-purple-50 rounded-lg border border-blue-200 flex items-center justify-between">
<div className="flex items-center gap-2">
{isLoading ? (
<>
<LoadingOutlined className="text-blue-500" spin />
<span className="text-sm text-blue-700 font-medium">Running Python code...</span>
</>
) : (
<>
<CodeOutlined className="text-blue-500" />
<span className="text-sm text-blue-700 font-medium">Code Interpreter Active</span>
</>
)}
</div>
<button
className="text-xs text-blue-500 hover:text-blue-700"
onClick={() => codeInterpreter.setEnabled(false)}
>
Disable
</button>
</div>
{/* Sample prompts - only show when not loading */}
{!isLoading && (
<div className="flex flex-wrap gap-2">
{[
"Generate sample sales data CSV and create a chart",
"Create a PNG bar chart comparing AI gateway providers including LiteLLM",
"Generate a CSV of LLM pricing data and visualize it as a line chart",
].map((prompt, idx) => (
<button
key={idx}
className="text-xs px-3 py-1.5 bg-white border border-gray-200 rounded-full hover:bg-blue-50 hover:border-blue-300 hover:text-blue-600 transition-colors"
onClick={() => setInputMessage(prompt)}
>
{prompt}
</button>
))}
</div>
)}
</div>
)}
<div className="flex items-center gap-2">
<div className="flex items-center flex-1 bg-white border border-gray-300 rounded-xl px-3 py-1 min-h-[44px]">
{/* Left: paperclip icon */}
<div className="flex-shrink-0 mr-2">
{/* Left: attachment and code interpreter icons */}
<div className="flex-shrink-0 mr-2 flex items-center gap-1">
{endpointType === EndpointType.RESPONSES && !responsesUploadedImage && (
<ResponsesImageUpload
responsesUploadedImage={responsesUploadedImage}
@@ -1792,6 +1880,26 @@ const ChatUI: React.FC<ChatUIProps> = ({
onRemoveImage={handleRemoveChatImage}
/>
)}
{/* Quick Code Interpreter toggle for Responses */}
{endpointType === EndpointType.RESPONSES && (
<Tooltip title={codeInterpreter.enabled ? "Code Interpreter enabled (click to disable)" : "Enable Code Interpreter"}>
<button
className={`p-1.5 rounded-md transition-colors ${
codeInterpreter.enabled
? "bg-blue-100 text-blue-600"
: "text-gray-400 hover:text-gray-600 hover:bg-gray-100"
}`}
onClick={() => {
codeInterpreter.toggle();
if (!codeInterpreter.enabled) {
NotificationsManager.success("Code Interpreter enabled!");
}
}}
>
<CodeOutlined style={{ fontSize: "16px" }} />
</button>
</Tooltip>
)}
</div>
{/* Middle: input field */}
@@ -0,0 +1,224 @@
import React, { useState, useEffect } from "react";
import { Collapse, Spin } from "antd";
import {
CodeOutlined,
DownloadOutlined,
FileImageOutlined,
FileTextOutlined,
LoadingOutlined,
} from "@ant-design/icons";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
import { coy } from "react-syntax-highlighter/dist/esm/styles/prism";
import { getProxyBaseUrl } from "@/components/networking";
interface ContainerFileCitation {
type: "container_file_citation";
container_id: string;
file_id: string;
filename: string;
start_index: number;
end_index: number;
}
interface CodeInterpreterOutputProps {
code?: string;
containerId?: string;
annotations?: ContainerFileCitation[];
accessToken: string;
}
const CodeInterpreterOutput: React.FC<CodeInterpreterOutputProps> = ({
code,
containerId,
annotations = [],
accessToken,
}) => {
const [imageUrls, setImageUrls] = useState<Record<string, string>>({});
const [loadingImages, setLoadingImages] = useState<Record<string, boolean>>({});
const proxyBaseUrl = getProxyBaseUrl();
// Fetch images from container files API
useEffect(() => {
const fetchImages = async () => {
for (const annotation of annotations) {
const isImage = annotation.filename?.toLowerCase().endsWith(".png") ||
annotation.filename?.toLowerCase().endsWith(".jpg") ||
annotation.filename?.toLowerCase().endsWith(".jpeg") ||
annotation.filename?.toLowerCase().endsWith(".gif");
if (isImage && annotation.container_id && annotation.file_id) {
setLoadingImages(prev => ({ ...prev, [annotation.file_id]: true }));
try {
// Fetch image content from container files API
const response = await fetch(
`${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
}
);
if (response.ok) {
const blob = await response.blob();
const url = URL.createObjectURL(blob);
setImageUrls(prev => ({ ...prev, [annotation.file_id]: url }));
}
} catch (error) {
console.error("Error fetching image:", error);
} finally {
setLoadingImages(prev => ({ ...prev, [annotation.file_id]: false }));
}
}
}
};
if (annotations.length > 0 && accessToken) {
fetchImages();
}
// Cleanup URLs on unmount
return () => {
Object.values(imageUrls).forEach(url => URL.revokeObjectURL(url));
};
}, [annotations, accessToken, proxyBaseUrl]);
const handleDownload = async (annotation: ContainerFileCitation) => {
try {
const response = await fetch(
`${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
}
);
if (response.ok) {
const blob = await response.blob();
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = annotation.filename || `file_${annotation.file_id}`;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
}
} catch (error) {
console.error("Error downloading file:", error);
}
};
// Separate images and other files
const imageAnnotations = annotations.filter(a =>
a.filename?.toLowerCase().endsWith(".png") ||
a.filename?.toLowerCase().endsWith(".jpg") ||
a.filename?.toLowerCase().endsWith(".jpeg") ||
a.filename?.toLowerCase().endsWith(".gif")
);
const fileAnnotations = annotations.filter(a =>
!a.filename?.toLowerCase().endsWith(".png") &&
!a.filename?.toLowerCase().endsWith(".jpg") &&
!a.filename?.toLowerCase().endsWith(".jpeg") &&
!a.filename?.toLowerCase().endsWith(".gif")
);
if (!code && annotations.length === 0) {
return null;
}
return (
<div className="mt-3 space-y-3">
{/* Executed Code - Collapsible */}
{code && (
<Collapse
size="small"
items={[
{
key: "code",
label: (
<span className="flex items-center gap-2 text-sm text-gray-600">
<CodeOutlined /> Python Code Executed
</span>
),
children: (
<SyntaxHighlighter
language="python"
style={coy}
customStyle={{
margin: 0,
borderRadius: "6px",
fontSize: "12px",
maxHeight: "300px",
overflow: "auto",
}}
>
{code}
</SyntaxHighlighter>
),
},
]}
/>
)}
{/* Generated Images */}
{imageAnnotations.map((annotation) => (
<div key={annotation.file_id} className="rounded-lg border border-gray-200 overflow-hidden">
{loadingImages[annotation.file_id] ? (
<div className="flex items-center justify-center p-8 bg-gray-50">
<Spin indicator={<LoadingOutlined spin />} />
<span className="ml-2 text-sm text-gray-500">Loading image...</span>
</div>
) : imageUrls[annotation.file_id] ? (
<div>
<img
src={imageUrls[annotation.file_id]}
alt={annotation.filename || "Generated chart"}
className="max-w-full"
style={{ maxHeight: "400px" }}
/>
<div className="flex items-center justify-between px-3 py-2 bg-gray-50 border-t border-gray-200">
<span className="text-xs text-gray-500 flex items-center gap-1">
<FileImageOutlined /> {annotation.filename}
</span>
<button
onClick={() => handleDownload(annotation)}
className="text-xs text-blue-500 hover:text-blue-700 flex items-center gap-1"
>
<DownloadOutlined /> Download
</button>
</div>
</div>
) : (
<div className="flex items-center justify-center p-4 bg-gray-50">
<span className="text-sm text-gray-400">Image not available</span>
</div>
)}
</div>
))}
{/* Download Links for Other Files */}
{fileAnnotations.length > 0 && (
<div className="flex flex-wrap gap-2">
{fileAnnotations.map((annotation) => (
<button
key={annotation.file_id}
onClick={() => handleDownload(annotation)}
className="flex items-center gap-2 px-3 py-2 bg-gray-50 border border-gray-200 rounded-lg hover:bg-gray-100 transition-colors"
>
<FileTextOutlined className="text-blue-500" />
<span className="text-sm">{annotation.filename}</span>
<DownloadOutlined className="text-gray-400" />
</button>
))}
</div>
)}
</div>
);
};
export default CodeInterpreterOutput;
@@ -0,0 +1,88 @@
import React from "react";
import { Switch, Tooltip, message } from "antd";
import { CodeOutlined, InfoCircleOutlined, ExclamationCircleOutlined } from "@ant-design/icons";
import { Text } from "@tremor/react";
interface CodeInterpreterToolProps {
accessToken: string;
enabled: boolean;
onEnabledChange: (enabled: boolean) => void;
selectedContainerId: string | null;
onContainerChange: (containerId: string | null) => void;
selectedModel: string;
disabled?: boolean;
}
const GITHUB_FEATURE_REQUEST_URL = "https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml";
const isOpenAIModel = (model: string): boolean => {
if (!model) return false;
const lowerModel = model.toLowerCase();
return (
lowerModel.startsWith("openai/") ||
lowerModel.startsWith("gpt-") ||
lowerModel.startsWith("o1") ||
lowerModel.startsWith("o3") ||
lowerModel.includes("openai")
);
};
const CodeInterpreterTool: React.FC<CodeInterpreterToolProps> = ({
enabled,
onEnabledChange,
selectedModel,
disabled = false,
}) => {
const isOpenAI = isOpenAIModel(selectedModel);
const isDisabled = disabled || !isOpenAI;
const handleToggle = (checked: boolean) => {
if (checked && !isOpenAI) {
message.warning("Code Interpreter is only available for OpenAI models");
return;
}
onEnabledChange(checked);
};
return (
<div className="border border-gray-200 rounded-lg p-3 bg-gradient-to-r from-blue-50 to-purple-50">
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<CodeOutlined className="text-blue-500" />
<Text className="font-medium text-gray-700">Code Interpreter</Text>
<Tooltip title="Run Python code to generate files, charts, and analyze data. Container is created automatically.">
<InfoCircleOutlined className="text-gray-400 text-xs" />
</Tooltip>
</div>
<Switch
checked={enabled && isOpenAI}
onChange={handleToggle}
disabled={isDisabled}
size="small"
className={enabled && isOpenAI ? "bg-blue-500" : ""}
/>
</div>
{!isOpenAI && (
<div className="mt-2 pt-2 border-t border-gray-200">
<div className="flex items-start gap-2">
<ExclamationCircleOutlined className="text-amber-500 mt-0.5" />
<div className="text-xs text-gray-600">
<span>Code Interpreter is currently only supported for OpenAI models. </span>
<a
href={GITHUB_FEATURE_REQUEST_URL}
target="_blank"
rel="noopener noreferrer"
className="text-blue-600 hover:text-blue-800 underline"
>
Request support for other providers
</a>
</div>
</div>
</div>
)}
</div>
);
};
export default CodeInterpreterTool;
@@ -0,0 +1,57 @@
/**
* Custom hook for managing Code Interpreter state.
* Container creation is handled automatically by OpenAI with container: { type: "auto" }
*/
import { useState, useCallback } from "react";
import { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler";
export interface UseCodeInterpreterReturn {
// State
enabled: boolean;
result: CodeInterpreterResult | null;
// Actions
setEnabled: (enabled: boolean) => void;
setResult: (result: CodeInterpreterResult | null) => void;
clearResult: () => void;
toggle: () => void;
}
export function useCodeInterpreter(): UseCodeInterpreterReturn {
const [enabled, setEnabledState] = useState<boolean>(() => {
if (typeof window === "undefined") return false;
const saved = sessionStorage.getItem("codeInterpreterEnabled");
return saved ? JSON.parse(saved) : false;
});
const [result, setResult] = useState<CodeInterpreterResult | null>(null);
// Persist enabled state to session storage
const setEnabled = useCallback((value: boolean) => {
setEnabledState(value);
if (typeof window !== "undefined") {
sessionStorage.setItem("codeInterpreterEnabled", JSON.stringify(value));
}
}, []);
const clearResult = useCallback(() => {
setResult(null);
}, []);
const toggle = useCallback(() => {
setEnabled(!enabled);
}, [enabled, setEnabled]);
return {
enabled,
result,
setEnabled,
setResult,
clearResult,
toggle,
};
}
// Re-export the type for convenience
export type { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler";
@@ -0,0 +1,83 @@
/**
* Code Interpreter event handling for the Responses API.
*/
export interface CodeInterpreterResult {
code: string;
containerId: string;
annotations: Array<{
type: "container_file_citation";
container_id: string;
file_id: string;
filename: string;
start_index: number;
end_index: number;
}>;
}
export interface CodeInterpreterState {
code: string;
containerId: string;
}
/**
* Handle code interpreter call completed event.
* Extracts code and container ID from the event.
*/
export function handleCodeInterpreterCall(
event: any,
state: CodeInterpreterState
): CodeInterpreterState {
if (event.type === "response.output_item.done" && event.item?.type === "code_interpreter_call") {
console.log("Code interpreter call completed:", event.item);
return {
code: event.item.code || "",
containerId: event.item.container_id || "",
};
}
return state;
}
/**
* Handle code interpreter output with file annotations.
* Calls the callback if file annotations are present.
*/
export function handleCodeInterpreterOutput(
event: any,
state: CodeInterpreterState,
onCodeInterpreterResult?: (result: CodeInterpreterResult) => void
): void {
if (
event.type === "response.output_item.done" &&
event.item?.type === "message" &&
event.item?.content &&
onCodeInterpreterResult
) {
const content = event.item.content;
for (const part of content) {
if (part.type === "output_text" && part.annotations) {
const fileAnnotations = part.annotations.filter(
(a: any) => a.type === "container_file_citation"
);
if (fileAnnotations.length > 0 || state.code) {
onCodeInterpreterResult({
code: state.code,
containerId: state.containerId,
annotations: fileAnnotations,
});
}
}
}
}
}
/**
* Check if code interpreter is being used based on event type.
*/
export function isCodeInterpreterEvent(event: any): boolean {
return (
event.type === "response.output_item.done" &&
event.item?.type === "code_interpreter_call"
);
}
@@ -4,6 +4,14 @@ import { TokenUsage } from "../chat_ui/ResponseMetrics";
import { getProxyBaseUrl } from "@/components/networking";
import NotificationManager from "@/components/molecules/notifications_manager";
import { MCPEvent } from "../chat_ui/MCPEventsDisplay";
import {
CodeInterpreterResult,
CodeInterpreterState,
handleCodeInterpreterCall,
handleCodeInterpreterOutput,
} from "./code_interpreter_handler";
export type { CodeInterpreterResult } from "./code_interpreter_handler";
export async function makeOpenAIResponsesRequest(
messages: MessageType[],
@@ -22,11 +30,17 @@ export async function makeOpenAIResponsesRequest(
previousResponseId?: string | null,
onResponseId?: (responseId: string) => void,
onMCPEvent?: (event: MCPEvent) => void,
codeInterpreterEnabled?: boolean,
onCodeInterpreterResult?: (result: CodeInterpreterResult) => void,
) {
if (!accessToken) {
throw new Error("Virtual Key is required");
}
if (!selectedModel || selectedModel.trim() === "") {
throw new Error("Model is required. Please select a model before sending a request.");
}
// Base URL should be the current base_url
const isLocal = process.env.NODE_ENV === "development";
if (isLocal !== true) {
@@ -69,19 +83,27 @@ export async function makeOpenAIResponsesRequest(
};
});
// Format MCP tools if selected
const tools =
selectedMCPTools && selectedMCPTools.length > 0
? [
{
type: "mcp",
server_label: "litellm",
server_url: `litellm_proxy/mcp`,
require_approval: "never",
allowed_tools: selectedMCPTools,
},
]
: undefined;
// Build tools array
const tools: any[] = [];
// Add MCP tools if selected
if (selectedMCPTools && selectedMCPTools.length > 0) {
tools.push({
type: "mcp",
server_label: "litellm",
server_url: `litellm_proxy/mcp`,
require_approval: "never",
allowed_tools: selectedMCPTools,
});
}
// Add code_interpreter tool if enabled (OpenAI auto-creates container)
if (codeInterpreterEnabled) {
tools.push({
type: "code_interpreter",
container: { type: "auto" },
});
}
// Create request to OpenAI responses API
// Use 'any' type to avoid TypeScript issues with the experimental API
@@ -94,12 +116,13 @@ export async function makeOpenAIResponsesRequest(
...(previousResponseId ? { previous_response_id: previousResponseId } : {}),
...(vector_store_ids ? { vector_store_ids } : {}),
...(guardrails ? { guardrails } : {}),
...(tools ? { tools, tool_choice: "required" } : {}),
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
},
{ signal },
);
let mcpToolUsed = "";
let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" };
for await (const event of response) {
console.log("Response event:", event);
@@ -137,6 +160,10 @@ export async function makeOpenAIResponsesRequest(
console.log("MCP tool used:", mcpToolUsed);
}
// Handle code interpreter events
codeInterpreterState = handleCodeInterpreterCall(event, codeInterpreterState);
handleCodeInterpreterOutput(event, codeInterpreterState, onCodeInterpreterResult);
// Handle output text delta
// 1) drop any "role" streams
if (event.type === "response.role.delta") {