diff --git a/litellm/containers/README.md b/litellm/containers/README.md new file mode 100644 index 0000000000..2b9fb5dec6 --- /dev/null +++ b/litellm/containers/README.md @@ -0,0 +1,241 @@ +# Container Files API + +This module provides a unified interface for container file operations across multiple LLM providers (OpenAI, Azure OpenAI, etc.). + +## Architecture + +``` +endpoints.json # Declarative endpoint definitions + ↓ +endpoint_factory.py # Auto-generates SDK functions + ↓ +container_handler.py # Generic HTTP handler + ↓ +BaseContainerConfig # Provider-specific transformations +├── OpenAIContainerConfig +└── AzureContainerConfig (example) +``` + +## Files Overview + +| File | Purpose | +|------|---------| +| `endpoints.json` | **Single source of truth** - Defines all container file endpoints | +| `endpoint_factory.py` | Auto-generates SDK functions (`list_container_files`, etc.) | +| `main.py` | Core container operations (create, list, retrieve, delete containers) | +| `utils.py` | Request parameter utilities | + +## Adding a New Endpoint + +To add a new container file endpoint (e.g., `get_container_file_content`): + +### Step 1: Add to `endpoints.json` + +```json +{ + "name": "get_container_file_content", + "async_name": "aget_container_file_content", + "path": "/containers/{container_id}/files/{file_id}/content", + "method": "GET", + "path_params": ["container_id", "file_id"], + "query_params": [], + "response_type": "ContainerFileContentResponse" +} +``` + +### Step 2: Add Response Type (if new) + +In `litellm/types/containers/main.py`: + +```python +class ContainerFileContentResponse(BaseModel): + """Response for file content download.""" + content: bytes + # ... other fields +``` + +### Step 3: Register Response Type + +In `litellm/llms/custom_httpx/container_handler.py`, add to `RESPONSE_TYPES`: + +```python +RESPONSE_TYPES = { + # ... existing types + "ContainerFileContentResponse": ContainerFileContentResponse, +} +``` + +### Step 4: Update Router (one-time setup) + +In `litellm/router.py`, add the call_type to the factory_function Literal and `_init_containers_api_endpoints` condition. + +In `litellm/proxy/route_llm_request.py`, add to the route mappings and skip-model-routing lists. + +### Step 5: Update Proxy Handler Factory (if new path params) + +If your endpoint has a new combination of path parameters, add a handler in `litellm/proxy/container_endpoints/handler_factory.py`: + +```python +elif path_params == ["container_id", "file_id", "new_param"]: + async def handler(...): + # handler implementation +``` + +--- + +## Adding a New Provider (e.g., Azure OpenAI) + +### Step 1: Create Provider Config + +Create `litellm/llms/azure/containers/transformation.py`: + +```python +from typing import Dict, Optional, Tuple, Any +import httpx + +from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.types.containers.main import ( + ContainerFileListResponse, + ContainerFileObject, + DeleteContainerFileResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.secret_managers.main import get_secret_str + + +class AzureContainerConfig(BaseContainerConfig): + """Configuration class for Azure OpenAI container API.""" + + def get_supported_openai_params(self) -> list: + return ["name", "expires_after", "file_ids", "extra_headers"] + + def map_openai_params( + self, + container_create_optional_params, + drop_params: bool, + ) -> Dict: + return dict(container_create_optional_params) + + def validate_environment( + self, + headers: dict, + api_key: Optional[str] = None, + ) -> dict: + """Azure uses api-key header instead of Bearer token.""" + import litellm + + api_key = ( + api_key + or litellm.azure_key + or get_secret_str("AZURE_API_KEY") + ) + headers["api-key"] = api_key + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Azure format: + https://{resource}.openai.azure.com/openai/containers?api-version=2024-xx + """ + if api_base is None: + raise ValueError("api_base is required for Azure") + + api_version = litellm_params.get("api_version", "2024-02-15-preview") + return f"{api_base.rstrip('/')}/openai/containers?api-version={api_version}" + + # Implement remaining abstract methods from BaseContainerConfig: + # - transform_container_create_request + # - transform_container_create_response + # - transform_container_list_request + # - transform_container_list_response + # - transform_container_retrieve_request + # - transform_container_retrieve_response + # - transform_container_delete_request + # - transform_container_delete_response + # - transform_container_file_list_request + # - transform_container_file_list_response +``` + +### Step 2: Register Provider Config + +In `litellm/utils.py`, find `ProviderConfigManager.get_provider_container_config()` and add: + +```python +@staticmethod +def get_provider_container_config( + provider: LlmProviders, +) -> Optional[BaseContainerConfig]: + if provider == LlmProviders.OPENAI: + from litellm.llms.openai.containers.transformation import OpenAIContainerConfig + return OpenAIContainerConfig() + elif provider == LlmProviders.AZURE: + from litellm.llms.azure.containers.transformation import AzureContainerConfig + return AzureContainerConfig() + return None +``` + +### Step 3: Test the New Provider + +```bash +# Create container via Azure +curl -X POST "http://localhost:4000/v1/containers" \ + -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: azure" \ + -H "Content-Type: application/json" \ + -d '{"name": "My Azure Container"}' + +# List container files via Azure +curl -X GET "http://localhost:4000/v1/containers/cntr_123/files" \ + -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: azure" +``` + +--- + +## How Provider Selection Works + +1. **Proxy receives request** with `custom-llm-provider` header/query/body +2. **Router calls** `ProviderConfigManager.get_provider_container_config(provider)` +3. **Generic handler** uses the provider config for: + - URL construction (`get_complete_url`) + - Authentication (`validate_environment`) + - Request/response transformation + +--- + +## Testing + +Run the container API tests: + +```bash +cd /Users/ishaanjaffer/github/litellm +python -m pytest tests/test_litellm/containers/ -v +``` + +Test via proxy: + +```bash +# Start proxy +cd litellm/proxy && python proxy_cli.py --config proxy_config.yaml --port 4000 + +# Test endpoints +curl -X GET "http://localhost:4000/v1/containers/cntr_123/files" \ + -H "Authorization: Bearer sk-1234" +``` + +--- + +## Endpoint Reference + +| Endpoint | Method | Path | +|----------|--------|------| +| List container files | GET | `/v1/containers/{container_id}/files` | +| Retrieve container file | GET | `/v1/containers/{container_id}/files/{file_id}` | +| Delete container file | DELETE | `/v1/containers/{container_id}/files/{file_id}` | + +See `endpoints.json` for the complete list. + diff --git a/litellm/containers/__init__.py b/litellm/containers/__init__.py index 0c32ea5c5b..e279cb429e 100644 --- a/litellm/containers/__init__.py +++ b/litellm/containers/__init__.py @@ -1,5 +1,16 @@ """Container management functions for LiteLLM.""" +# Auto-generated container file functions from endpoints.json +from .endpoint_factory import ( + adelete_container_file, + alist_container_files, + aretrieve_container_file, + aretrieve_container_file_content, + delete_container_file, + list_container_files, + retrieve_container_file, + retrieve_container_file_content, +) from .main import ( acreate_container, adelete_container, @@ -12,6 +23,7 @@ from .main import ( ) __all__ = [ + # Core container operations "acreate_container", "adelete_container", "alist_containers", @@ -20,5 +32,14 @@ __all__ = [ "delete_container", "list_containers", "retrieve_container", + # Container file operations (auto-generated from endpoints.json) + "adelete_container_file", + "alist_container_files", + "aretrieve_container_file", + "aretrieve_container_file_content", + "delete_container_file", + "list_container_files", + "retrieve_container_file", + "retrieve_container_file_content", ] diff --git a/litellm/containers/endpoint_factory.py b/litellm/containers/endpoint_factory.py new file mode 100644 index 0000000000..998b42a3ab --- /dev/null +++ b/litellm/containers/endpoint_factory.py @@ -0,0 +1,224 @@ +""" +Factory for generating container SDK functions from JSON config. + +This module reads endpoints.json and dynamically generates SDK functions +that use the generic container handler. +""" + +import asyncio +import contextvars +import json +from functools import partial +from pathlib import Path +from typing import Any, Callable, Dict, List, Literal, Optional, Type + +import litellm +from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.llms.custom_httpx.container_handler import generic_container_handler +from litellm.types.containers.main import ( + ContainerFileListResponse, + ContainerFileObject, + DeleteContainerFileResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.utils import ProviderConfigManager, client + +# Response type mapping +RESPONSE_TYPES: Dict[str, Type] = { + "ContainerFileListResponse": ContainerFileListResponse, + "ContainerFileObject": ContainerFileObject, + "DeleteContainerFileResponse": DeleteContainerFileResponse, +} + + +def _load_endpoints_config() -> Dict: + """Load the endpoints configuration from JSON file.""" + config_path = Path(__file__).parent / "endpoints.json" + with open(config_path) as f: + return json.load(f) + + +def create_sync_endpoint_function(endpoint_config: Dict) -> Callable: + """ + Create a sync SDK function from endpoint config. + + Uses the generic container handler instead of individual handler methods. + """ + endpoint_name = endpoint_config["name"] + response_type = RESPONSE_TYPES.get(endpoint_config["response_type"]) + path_params = endpoint_config.get("path_params", []) + + @client + def endpoint_func( + timeout: int = 600, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, + ): + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id") + _is_async = kwargs.pop("async_call", False) is True + + # Check for mock response + mock_response = kwargs.get("mock_response") + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + if response_type: + return response_type(**mock_response) + return mock_response + + # Get provider config + litellm_params = GenericLiteLLMParams(**kwargs) + container_provider_config: Optional[BaseContainerConfig] = ( + ProviderConfigManager.get_provider_container_config( + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if container_provider_config is None: + raise ValueError(f"Container provider config not found for: {custom_llm_provider}") + + # Build optional params for logging + optional_params = {k: kwargs.get(k) for k in path_params if k in kwargs} + + # Pre-call logging + litellm_logging_obj.update_environment_variables( + model="", + optional_params=optional_params, + litellm_params={"litellm_call_id": litellm_call_id}, + custom_llm_provider=custom_llm_provider, + ) + + # Use generic handler + return generic_container_handler.handle( + endpoint_name=endpoint_name, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + **kwargs, + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + return endpoint_func + + +def create_async_endpoint_function( + sync_func: Callable, + endpoint_config: Dict, +) -> Callable: + """Create an async SDK function that wraps the sync function.""" + + @client + async def async_endpoint_func( + timeout: int = 600, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, + ): + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + sync_func, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + return async_endpoint_func + + +def generate_container_endpoints() -> Dict[str, Callable]: + """ + Generate all container endpoint functions from the JSON config. + + Returns a dict mapping function names to their implementations. + """ + config = _load_endpoints_config() + endpoints = {} + + for endpoint_config in config["endpoints"]: + # Create sync function + sync_func = create_sync_endpoint_function(endpoint_config) + endpoints[endpoint_config["name"]] = sync_func + + # Create async function + async_func = create_async_endpoint_function(sync_func, endpoint_config) + endpoints[endpoint_config["async_name"]] = async_func + + return endpoints + + +def get_all_endpoint_names() -> List[str]: + """Get all endpoint names (sync and async) from config.""" + config = _load_endpoints_config() + names = [] + for endpoint in config["endpoints"]: + names.append(endpoint["name"]) + names.append(endpoint["async_name"]) + return names + + +def get_async_endpoint_names() -> List[str]: + """Get all async endpoint names for router registration.""" + config = _load_endpoints_config() + return [endpoint["async_name"] for endpoint in config["endpoints"]] + + +# Generate endpoints on module load +_generated_endpoints = generate_container_endpoints() + +# Export generated functions dynamically +list_container_files = _generated_endpoints.get("list_container_files") +alist_container_files = _generated_endpoints.get("alist_container_files") +retrieve_container_file = _generated_endpoints.get("retrieve_container_file") +aretrieve_container_file = _generated_endpoints.get("aretrieve_container_file") +delete_container_file = _generated_endpoints.get("delete_container_file") +adelete_container_file = _generated_endpoints.get("adelete_container_file") +retrieve_container_file_content = _generated_endpoints.get("retrieve_container_file_content") +aretrieve_container_file_content = _generated_endpoints.get("aretrieve_container_file_content") diff --git a/litellm/containers/endpoints.json b/litellm/containers/endpoints.json new file mode 100644 index 0000000000..4a23fc75c3 --- /dev/null +++ b/litellm/containers/endpoints.json @@ -0,0 +1,41 @@ +{ + "endpoints": [ + { + "name": "list_container_files", + "async_name": "alist_container_files", + "path": "/containers/{container_id}/files", + "method": "GET", + "path_params": ["container_id"], + "query_params": ["after", "limit", "order"], + "response_type": "ContainerFileListResponse" + }, + { + "name": "retrieve_container_file", + "async_name": "aretrieve_container_file", + "path": "/containers/{container_id}/files/{file_id}", + "method": "GET", + "path_params": ["container_id", "file_id"], + "query_params": [], + "response_type": "ContainerFileObject" + }, + { + "name": "delete_container_file", + "async_name": "adelete_container_file", + "path": "/containers/{container_id}/files/{file_id}", + "method": "DELETE", + "path_params": ["container_id", "file_id"], + "query_params": [], + "response_type": "DeleteContainerFileResponse" + }, + { + "name": "retrieve_container_file_content", + "async_name": "aretrieve_container_file_content", + "path": "/containers/{container_id}/files/{file_id}/content", + "method": "GET", + "path_params": ["container_id", "file_id"], + "query_params": [], + "response_type": "raw", + "returns_binary": true + } + ] +} diff --git a/litellm/containers/main.py b/litellm/containers/main.py index c499f945d6..1fe7a26c0a 100644 --- a/litellm/containers/main.py +++ b/litellm/containers/main.py @@ -12,6 +12,7 @@ from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.main import base_llm_http_handler from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, + ContainerFileListResponse, ContainerListOptionalRequestParams, ContainerListResponse, ContainerObject, @@ -24,10 +25,12 @@ from litellm.utils import ProviderConfigManager, client __all__ = [ "acreate_container", "adelete_container", + "alist_container_files", "alist_containers", "aretrieve_container", "create_container", "delete_container", + "list_container_files", "list_containers", "retrieve_container", ] @@ -147,6 +150,9 @@ def create_container( expires_after: Optional[Dict[str, Any]] = None, file_ids: Optional[List[str]] = None, timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, custom_llm_provider: Literal["openai"] = "openai", # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -362,6 +368,9 @@ def list_containers( limit: Optional[int] = None, order: Optional[str] = None, timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, custom_llm_provider: Literal["openai"] = "openai", # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -547,6 +556,9 @@ def retrieve_container( def retrieve_container( container_id: str, timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, custom_llm_provider: Literal["openai"] = "openai", # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -724,6 +736,9 @@ def delete_container( def delete_container( container_id: str, timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, custom_llm_provider: Literal["openai"] = "openai", # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -799,3 +814,200 @@ def delete_container( extra_kwargs=kwargs, ) + +##### Container Files List ####################### +@client +async def alist_container_files( + container_id: str, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + timeout=600, # default to 10 minutes + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> ContainerFileListResponse: + """Asynchronously list files in a container. + + Parameters: + - `container_id` (str): The ID of the container + - `after` (Optional[str]): A cursor for pagination + - `limit` (Optional[int]): Number of items to return (1-100, default 20) + - `order` (Optional[str]): Sort order ('asc' or 'desc', default 'desc') + - `timeout` (int): Request timeout in seconds + - `custom_llm_provider` (Literal["openai"]): The LLM provider to use + - `extra_headers` (Optional[Dict[str, Any]]): Additional headers + - `extra_query` (Optional[Dict[str, Any]]): Additional query parameters + - `extra_body` (Optional[Dict[str, Any]]): Additional body parameters + - `kwargs` (dict): Additional keyword arguments + + Returns: + - `response` (ContainerFileListResponse): The list of container files + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + list_container_files, + container_id=container_id, + after=after, + limit=limit, + order=order, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +# fmt: off + +@overload +def list_container_files( + container_id: str, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + timeout=600, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + *, + alist_container_files: Literal[True], + **kwargs, +) -> Coroutine[Any, Any, ContainerFileListResponse]: + ... + + +@overload +def list_container_files( + container_id: str, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + timeout=600, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + *, + alist_container_files: Literal[False] = False, + **kwargs, +) -> ContainerFileListResponse: + ... + +# fmt: on + + +@client +def list_container_files( + container_id: str, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[ + ContainerFileListResponse, + Coroutine[Any, Any, ContainerFileListResponse], +]: + """List files in a container using the OpenAI Container API. + + Currently supports OpenAI + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id") + _is_async = kwargs.pop("async_call", False) is True + + # Check for mock response first + mock_response = kwargs.get("mock_response") + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + + response = ContainerFileListResponse(**mock_response) + return response + + # get llm provider logic + litellm_params = GenericLiteLLMParams(**kwargs) + # get provider config + container_provider_config: Optional[BaseContainerConfig] = ( + ProviderConfigManager.get_provider_container_config( + provider=litellm.LlmProviders(custom_llm_provider), + ) + ) + + if container_provider_config is None: + raise ValueError(f"Container provider config not found for provider: {custom_llm_provider}") + + # Pre Call logging + litellm_logging_obj.update_environment_variables( + model="", + optional_params={"container_id": container_id, "after": after, "limit": limit, "order": order}, + litellm_params={ + "litellm_call_id": litellm_call_id, + }, + custom_llm_provider=custom_llm_provider, + ) + + # Set the correct call type + litellm_logging_obj.call_type = CallTypes.list_container_files.value + + return base_llm_http_handler.container_file_list_handler( + container_id=container_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + after=after, + limit=limit, + order=order, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + diff --git a/litellm/llms/base_llm/containers/transformation.py b/litellm/llms/base_llm/containers/transformation.py index 429f5a76e2..5ce374c773 100644 --- a/litellm/llms/base_llm/containers/transformation.py +++ b/litellm/llms/base_llm/containers/transformation.py @@ -12,11 +12,12 @@ from litellm.types.router import GenericLiteLLMParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.types.containers.main import ( - ContainerListResponse as _ContainerListResponse, + ContainerFileListResponse as _ContainerFileListResponse, ) from litellm.types.containers.main import ( - ContainerObject as _ContainerObject, + ContainerListResponse as _ContainerListResponse, ) + from litellm.types.containers.main import ContainerObject as _ContainerObject from litellm.types.containers.main import ( DeleteContainerResult as _DeleteContainerResult, ) @@ -28,12 +29,14 @@ if TYPE_CHECKING: ContainerObject = _ContainerObject DeleteContainerResult = _DeleteContainerResult ContainerListResponse = _ContainerListResponse + ContainerFileListResponse = _ContainerFileListResponse else: LiteLLMLoggingObj = Any BaseLLMException = Any ContainerObject = Any DeleteContainerResult = Any ContainerListResponse = Any + ContainerFileListResponse = Any class BaseContainerConfig(ABC): @@ -193,6 +196,63 @@ class BaseContainerConfig(ABC): """Transform the container delete response.""" ... + @abstractmethod + def transform_container_file_list_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, Any] | None = None, + ) -> tuple[str, dict]: + """Transform the container file list request into a URL and params. + + Returns: + tuple[str, dict]: (url, params) for the container file list request. + """ + ... + + @abstractmethod + def transform_container_file_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerFileListResponse: + """Transform the container file list response.""" + ... + + @abstractmethod + def transform_container_file_content_request( + self, + container_id: str, + file_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[str, dict]: + """Transform the container file content request into a URL and params. + + Returns: + tuple[str, dict]: (url, params) for the container file content request. + """ + ... + + @abstractmethod + def transform_container_file_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + """Transform the container file content response. + + Returns: + bytes: The raw file content. + """ + ... + def get_error_class( self, error_message: str, diff --git a/litellm/llms/custom_httpx/container_handler.py b/litellm/llms/custom_httpx/container_handler.py new file mode 100644 index 0000000000..ed112e4dd5 --- /dev/null +++ b/litellm/llms/custom_httpx/container_handler.py @@ -0,0 +1,348 @@ +""" +Generic container file handler for LiteLLM. + +This module provides a single generic handler that can process any container file +endpoint defined in endpoints.json, eliminating the need for individual handler methods. +""" + +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union + +import httpx + +import litellm +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.types.containers.main import ( + ContainerFileListResponse, + ContainerFileObject, + DeleteContainerFileResponse, +) +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + + +# Response type mapping +RESPONSE_TYPES: Dict[str, Type] = { + "ContainerFileListResponse": ContainerFileListResponse, + "ContainerFileObject": ContainerFileObject, + "DeleteContainerFileResponse": DeleteContainerFileResponse, +} + + +def _load_endpoints_config() -> Dict: + """Load the endpoints configuration from JSON file.""" + config_path = Path(__file__).parent.parent.parent / "containers" / "endpoints.json" + with open(config_path) as f: + return json.load(f) + + +def _get_endpoint_config(endpoint_name: str) -> Optional[Dict]: + """Get config for a specific endpoint by name.""" + config = _load_endpoints_config() + for endpoint in config["endpoints"]: + if endpoint["name"] == endpoint_name or endpoint["async_name"] == endpoint_name: + return endpoint + return None + + +def _build_url( + api_base: str, + path_template: str, + path_params: Dict[str, str], +) -> str: + """Build the full URL by substituting path parameters. + + The api_base from get_complete_url already includes /containers, + so we need to strip that prefix from the path_template. + """ + # api_base ends with /containers, path_template starts with /containers + # So we need to strip /containers from the path + if path_template.startswith("/containers"): + path_template = path_template[len("/containers"):] + + url = f"{api_base.rstrip('/')}{path_template}" + for param, value in path_params.items(): + url = url.replace(f"{{{param}}}", value) + return url + + +def _build_query_params( + query_param_names: list, + kwargs: Dict[str, Any], +) -> Dict[str, str]: + """Build query parameters from kwargs.""" + params = {} + for param_name in query_param_names: + value = kwargs.get(param_name) + if value is not None: + params[param_name] = str(value) if not isinstance(value, str) else value + return params + + +class GenericContainerHandler: + """ + Generic handler for container file API endpoints. + + This single handler can process any endpoint defined in endpoints.json, + eliminating the need for individual handler methods per endpoint. + """ + + def handle( + self, + endpoint_name: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + **kwargs, + ) -> Union[Any, Coroutine[Any, Any, Any]]: + """ + Generic handler for any container file endpoint. + + Args: + endpoint_name: Name of the endpoint (e.g., "list_container_files") + container_provider_config: Provider-specific configuration + litellm_params: LiteLLM parameters including api_key, api_base + logging_obj: Logging object for request logging + extra_headers: Additional HTTP headers + extra_query: Additional query parameters + timeout: Request timeout + _is_async: Whether to make async request + client: Optional HTTP client + **kwargs: Path params and query params (e.g., container_id, file_id, after, limit) + """ + if _is_async: + return self._async_handle( + endpoint_name=endpoint_name, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + **kwargs, + ) + + return self._sync_handle( + endpoint_name=endpoint_name, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + **kwargs, + ) + + def _sync_handle( + self, + endpoint_name: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + **kwargs, + ) -> Any: + """Synchronous request handler.""" + endpoint_config = _get_endpoint_config(endpoint_name) + if not endpoint_config: + raise ValueError(f"Unknown endpoint: {endpoint_name}") + + # Get HTTP client + if client is None or not isinstance(client, HTTPHandler): + http_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + http_client = client + + # Build request + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + if extra_headers: + headers.update(extra_headers) + + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Build URL with path params + path_params = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])} + url = _build_url(api_base, endpoint_config["path"], path_params) + + # Build query params + query_params = _build_query_params(endpoint_config.get("query_params", []), kwargs) + if extra_query: + query_params.update(extra_query) + + # Log request + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": query_params, + }, + ) + + # Make request + method = endpoint_config["method"].upper() + returns_binary = endpoint_config.get("returns_binary", False) + + try: + if method == "GET": + response = http_client.get(url=url, headers=headers, params=query_params) + elif method == "DELETE": + response = http_client.delete(url=url, headers=headers, params=query_params) + elif method == "POST": + response = http_client.post(url=url, headers=headers, params=query_params) + else: + raise ValueError(f"Unsupported HTTP method: {method}") + + # For binary responses, return raw content + if returns_binary: + return response.content + + # Check for error response + response_json = response.json() + if "error" in response_json: + from litellm.llms.base_llm.chat.transformation import BaseLLMException + error_msg = response_json.get("error", {}).get("message", str(response_json)) + raise BaseLLMException( + status_code=response.status_code, + message=error_msg, + headers=dict(response.headers), + ) + + # Parse response + response_type = RESPONSE_TYPES.get(endpoint_config["response_type"]) + if response_type: + return response_type(**response_json) + return response_json + + except Exception as e: + raise e + + async def _async_handle( + self, + endpoint_name: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + **kwargs, + ) -> Any: + """Asynchronous request handler.""" + endpoint_config = _get_endpoint_config(endpoint_name) + if not endpoint_config: + raise ValueError(f"Unknown endpoint: {endpoint_name}") + + # Get HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + http_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + http_client = client + + # Build request + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + if extra_headers: + headers.update(extra_headers) + + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Build URL with path params + path_params = {p: kwargs.get(p, "") for p in endpoint_config.get("path_params", [])} + url = _build_url(api_base, endpoint_config["path"], path_params) + + # Build query params + query_params = _build_query_params(endpoint_config.get("query_params", []), kwargs) + if extra_query: + query_params.update(extra_query) + + # Log request + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": query_params, + }, + ) + + # Make request + method = endpoint_config["method"].upper() + returns_binary = endpoint_config.get("returns_binary", False) + + try: + if method == "GET": + response = await http_client.get(url=url, headers=headers, params=query_params) + elif method == "DELETE": + response = await http_client.delete(url=url, headers=headers, params=query_params) + elif method == "POST": + response = await http_client.post(url=url, headers=headers, params=query_params) + else: + raise ValueError(f"Unsupported HTTP method: {method}") + + # For binary responses, return raw content + if returns_binary: + return response.content + + # Check for error response + response_json = response.json() + if "error" in response_json: + from litellm.llms.base_llm.chat.transformation import BaseLLMException + error_msg = response_json.get("error", {}).get("message", str(response_json)) + raise BaseLLMException( + status_code=response.status_code, + message=error_msg, + headers=dict(response.headers), + ) + + # Parse response + response_type = RESPONSE_TYPES.get(endpoint_config["response_type"]) + if response_type: + return response_type(**response_json) + return response_json + + except Exception as e: + raise e + + +# Singleton instance +generic_container_handler = GenericContainerHandler() + diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 701cefb771..7a8b5ec357 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -66,6 +66,7 @@ from litellm.responses.streaming_iterator import ( SyncResponsesAPIStreamingIterator, ) from litellm.types.containers.main import ( + ContainerFileListResponse, ContainerListResponse, ContainerObject, DeleteContainerResult, @@ -5711,6 +5712,337 @@ class BaseLLMHTTPHandler: provider_config=container_provider_config, ) + def container_file_list_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union["ContainerFileListResponse", Coroutine[Any, Any, "ContainerFileListResponse"]]: + if _is_async: + return self.async_container_file_list_handler( + container_id=container_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + after=after, + limit=limit, + order=order, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for container files + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_file_list_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + after=after, + limit=limit, + order=order, + extra_query=extra_query, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_file_list_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_file_list_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "ContainerFileListResponse": + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for container files + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_file_list_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + after=after, + limit=limit, + order=order, + extra_query=extra_query, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_file_list_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + def container_file_content_handler( + self, + container_id: str, + file_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + if _is_async: + return self.async_container_file_content_handler( + container_id=container_id, + file_id=file_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for container files + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_file_content_request( + container_id=container_id, + file_id=file_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_file_content_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_file_content_handler( + self, + container_id: str, + file_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> bytes: + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for container files + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_file_content_request( + container_id=container_id, + file_id=file_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_file_content_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + ###### VECTOR STORE HANDLER ###### async def async_vector_store_search_handler( self, diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index 1a6343d7be..46718816f3 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( from litellm.secret_managers.main import get_secret_str from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, + ContainerFileListResponse, ContainerListResponse, ContainerObject, DeleteContainerResult, @@ -19,7 +20,9 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException - from ...base_llm.containers.transformation import BaseContainerConfig as _BaseContainerConfig + from ...base_llm.containers.transformation import ( + BaseContainerConfig as _BaseContainerConfig, + ) LiteLLMLoggingObj = _LiteLLMLoggingObj BaseContainerConfig = _BaseContainerConfig @@ -247,6 +250,86 @@ class OpenAIContainerConfig(BaseContainerConfig): return delete_result + def transform_container_file_list_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_query: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """Transform the container file list request for OpenAI API. + + OpenAI API expects the following request: + - GET /v1/containers/{container_id}/files + """ + # Construct the URL for container files + url = f"{api_base.rstrip('/')}/{container_id}/files" + + # Prepare query parameters + params: Dict[str, Any] = {} + if after is not None: + params["after"] = after + if limit is not None: + params["limit"] = str(limit) + if order is not None: + params["order"] = order + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + return url, params + + def transform_container_file_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerFileListResponse: + """Transform the OpenAI container file list response. + """ + response_data = raw_response.json() + + # Transform the response data + file_list = ContainerFileListResponse(**response_data) # type: ignore[arg-type] + + return file_list + + def transform_container_file_content_request( + self, + container_id: str, + file_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform the container file content request for OpenAI API. + + OpenAI API expects the following request: + - GET /v1/containers/{container_id}/files/{file_id}/content + """ + # Construct the URL for container file content + url = f"{api_base.rstrip('/')}/{container_id}/files/{file_id}/content" + + # No query parameters needed + params: Dict[str, Any] = {} + + return url, params + + def transform_container_file_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + """Transform the OpenAI container file content response. + + Returns the raw binary content of the file. + """ + return raw_response.content + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers], ) -> BaseLLMException: diff --git a/litellm/proxy/container_endpoints/endpoints.py b/litellm/proxy/container_endpoints/endpoints.py index 1a3d41f0ac..8581603eea 100644 --- a/litellm/proxy/container_endpoints/endpoints.py +++ b/litellm/proxy/container_endpoints/endpoints.py @@ -1,6 +1,7 @@ #### Container Endpoints ##### from typing import Any, Dict + from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse @@ -9,9 +10,9 @@ from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_au from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.common_utils.openai_endpoint_utils import ( + get_custom_llm_provider_from_request_body, get_custom_llm_provider_from_request_headers, get_custom_llm_provider_from_request_query, - get_custom_llm_provider_from_request_body, ) router = APIRouter() @@ -404,3 +405,10 @@ async def delete_container( version=version, ) + +# Register JSON-configured container file endpoints +from litellm.proxy.container_endpoints.handler_factory import ( + register_container_file_endpoints, +) + +register_container_file_endpoints(router) diff --git a/litellm/proxy/container_endpoints/handler_factory.py b/litellm/proxy/container_endpoints/handler_factory.py new file mode 100644 index 0000000000..7eee44afb4 --- /dev/null +++ b/litellm/proxy/container_endpoints/handler_factory.py @@ -0,0 +1,310 @@ +""" +Factory for generating container proxy endpoints from JSON config. + +This module reads the endpoints.json config and dynamically creates +FastAPI route handlers for ALL container file endpoints. +""" + +import json +from pathlib import Path +from typing import Any, Dict, List + +from fastapi import APIRouter, Depends, Request, Response +from fastapi.responses import ORJSONResponse + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.openai_endpoint_utils import ( + get_custom_llm_provider_from_request_headers, + get_custom_llm_provider_from_request_query, +) + + +def _load_endpoints_config() -> Dict: + """Load the endpoints configuration from JSON file.""" + config_path = Path(__file__).parent.parent.parent / "containers" / "endpoints.json" + with open(config_path) as f: + return json.load(f) + + +def get_all_route_types() -> List[str]: + """Get all async route types for registration in route_llm_request.py""" + config = _load_endpoints_config() + return [endpoint["async_name"] for endpoint in config["endpoints"]] + + +def _get_container_provider_config(custom_llm_provider: str): + """Get the container provider config for the given provider.""" + if custom_llm_provider == "openai": + from litellm.llms.openai.containers.transformation import OpenAIContainerConfig + return OpenAIContainerConfig() + else: + raise ValueError(f"Container API not supported for provider: {custom_llm_provider}") + + +def _create_handler_for_path_params(path_params: List[str], route_type: str, returns_binary: bool = False): + """ + Dynamically create a handler with the correct path parameter signature. + """ + # For binary content endpoints, use a different handler + if returns_binary and path_params == ["container_id", "file_id"]: + async def handler_binary_content( + request: Request, + container_id: str, + file_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + return await _process_binary_request( + request=request, + container_id=container_id, + file_id=file_id, + user_api_key_dict=user_api_key_dict, + ) + return handler_binary_content + + # Create handlers for different path parameter combinations + if path_params == ["container_id"]: + async def handler_container_id( + request: Request, + container_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + return await _process_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, + path_params={"container_id": container_id}, + ) + return handler_container_id + + elif path_params == ["container_id", "file_id"]: + async def handler_container_file( + request: Request, + container_id: str, + file_id: str, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + return await _process_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, + path_params={"container_id": container_id, "file_id": file_id}, + ) + return handler_container_file + + else: + # Fallback for no path params + async def handler_no_params( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + ): + return await _process_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, + path_params={}, + ) + return handler_no_params + + +async def _process_binary_request( + request: Request, + container_id: str, + file_id: str, + user_api_key_dict: UserAPIKeyAuth, +): + """ + Process binary content requests using the proper transformation pattern. + + This uses the provider config transformations and llm_http_handler + to maintain consistency with the established pattern. + """ + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + from litellm.types.router import GenericLiteLLMParams + + # Extract custom_llm_provider + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or "openai" + ) + + # Get the provider config + container_provider_config = _get_container_provider_config(custom_llm_provider) + + # Build litellm_params - credentials are resolved by provider config from env + litellm_params = GenericLiteLLMParams() + + # Create logging object + logging_obj = Logging( + model="container-file-content", + messages=[], + stream=False, + call_type="container_file_content", + start_time=None, + litellm_call_id="", + function_id="", + ) + + # Use the HTTP handler to make the request + handler = BaseLLMHTTPHandler() + + try: + content = await handler.async_container_file_content_handler( + container_id=container_id, + file_id=file_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + ) + + # Determine content type based on common file extensions in the file_id + content_type = "application/octet-stream" + file_id_lower = file_id.lower() + if ".png" in file_id_lower or file_id_lower.endswith("png"): + content_type = "image/png" + elif ".jpg" in file_id_lower or ".jpeg" in file_id_lower: + content_type = "image/jpeg" + elif ".gif" in file_id_lower: + content_type = "image/gif" + elif ".csv" in file_id_lower: + content_type = "text/csv" + elif ".json" in file_id_lower: + content_type = "application/json" + elif ".txt" in file_id_lower: + content_type = "text/plain" + elif ".pdf" in file_id_lower: + content_type = "application/pdf" + + return Response( + content=content, + media_type=content_type, + ) + + except Exception as e: + raise e + + +async def _process_request( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + route_type: str, + path_params: Dict[str, str], +): + """Common request processing logic.""" + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + query_params = dict(request.query_params) + data: Dict[str, Any] = { + "query_params": query_params, + **path_params, + } + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type=route_type, # type: ignore[arg-type] + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +def register_container_file_endpoints(router: APIRouter) -> None: + """ + Register ALL container file endpoints from JSON config to the router. + + This single function registers all endpoints defined in endpoints.json, + eliminating the need for manual endpoint definitions. + """ + config = _load_endpoints_config() + + for endpoint_config in config["endpoints"]: + path = endpoint_config["path"] + method = endpoint_config["method"].lower() + path_params = endpoint_config.get("path_params", []) + route_type = endpoint_config["async_name"] + returns_binary = endpoint_config.get("returns_binary", False) + + # Create handler with correct signature for path params + handler = _create_handler_for_path_params(path_params, route_type, returns_binary) + + # Register routes + route_method = getattr(router, method) + + # For binary endpoints, don't use ORJSONResponse + if returns_binary: + # Register both /v1/... and /... paths without JSON response class + route_method( + f"/v1{path}", + dependencies=[Depends(user_api_key_auth)], + tags=["containers"], + )(handler) + + route_method( + path, + dependencies=[Depends(user_api_key_auth)], + tags=["containers"], + )(handler) + else: + # Register both /v1/... and /... paths with JSON response + route_method( + f"/v1{path}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["containers"], + )(handler) + + route_method( + path, + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["containers"], + )(handler) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 221aa16f91..e55c71d164 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -36,6 +36,11 @@ ROUTE_ENDPOINT_MAPPING = { "alist_containers": "/containers", "aretrieve_container": "/containers/{container_id}", "adelete_container": "/containers/{container_id}", + # Auto-generated container file routes + "alist_container_files": "/containers/{container_id}/files", + "aretrieve_container_file": "/containers/{container_id}/files/{file_id}", + "adelete_container_file": "/containers/{container_id}/files/{file_id}", + "aretrieve_container_file_content": "/containers/{container_id}/files/{file_id}/content", "acreate_skill": "/skills", "alist_skills": "/skills", "aget_skill": "/skills/{skill_id}", @@ -132,6 +137,10 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", "acreate_skill", "alist_skills", "aget_skill", @@ -184,6 +193,10 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", ]: return getattr(llm_router, f"{route_type}")(**data) if route_type in [ @@ -256,6 +269,10 @@ async def route_request( "alist_containers", "aretrieve_container", "adelete_container", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", ]: # moderation endpoint does not require `model` parameter return getattr(llm_router, f"{route_type}")(**data) diff --git a/litellm/router.py b/litellm/router.py index eb83c430e6..96df0afd6c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1017,6 +1017,9 @@ class Router: list_containers, retrieve_container, ) + from litellm.containers.endpoint_factory import ( + _generated_endpoints as container_file_endpoints, + ) self.acreate_container = self.factory_function( acreate_container, call_type="acreate_container" @@ -1042,6 +1045,10 @@ class Router: self.delete_container = self.factory_function( delete_container, call_type="delete_container" ) + + # Auto-register JSON-generated container file endpoints + for name, func in container_file_endpoints.items(): + setattr(self, name, self.factory_function(func, call_type=name)) # type: ignore[arg-type] def _initialize_skills_endpoints(self): """Initialize Anthropic Skills API endpoints.""" @@ -3841,6 +3848,12 @@ class Router: "retrieve_container", "adelete_container", "delete_container", + "alist_container_files", + "list_container_files", + "aretrieve_container_file", + "retrieve_container_file", + "adelete_container_file", + "delete_container_file", "acreate_skill", "alist_skills", "aget_skill", @@ -3976,6 +3989,10 @@ class Router: "alist_containers", "aretrieve_container", "adelete_container", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", ): return await self._init_containers_api_endpoints( original_function=original_function, diff --git a/litellm/types/containers/main.py b/litellm/types/containers/main.py index 14c191e220..66aa7dc5fa 100644 --- a/litellm/types/containers/main.py +++ b/litellm/types/containers/main.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Literal, Optional -from typing_extensions import TypedDict from pydantic import BaseModel +from typing_extensions import TypedDict class ExpiresAfter(BaseModel): @@ -120,3 +120,76 @@ class ContainerListOptionalRequestParams(TypedDict, total=False): extra_headers: Optional[Dict[str, str]] extra_query: Optional[Dict[str, str]] + +class ContainerFileObject(BaseModel): + """Represents a container file object.""" + id: str + object: Literal["container.file", "container_file"] # OpenAI returns "container.file" + container_id: str + bytes: Optional[int] = None # Can be null for some files + created_at: int + path: str + source: str + _hidden_params: Dict[str, Any] = {} + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def json(self, **kwargs): # type: ignore + try: + return self.model_dump(**kwargs) + except Exception: + return self.dict() + + +class ContainerFileListResponse(BaseModel): + """Response object for list container files request.""" + object: Literal["list"] + data: List[ContainerFileObject] + first_id: Optional[str] = None + last_id: Optional[str] = None + has_more: bool + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def json(self, **kwargs): # type: ignore + try: + return self.model_dump(**kwargs) + except Exception: + return self.dict() + + +class DeleteContainerFileResponse(BaseModel): + """Response object for delete container file request.""" + id: str + object: Literal["container_file.deleted"] + deleted: bool + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def json(self, **kwargs): # type: ignore + try: + return self.model_dump(**kwargs) + except Exception: + return self.dict() + diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2ccc14a271..d788f2840e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -319,6 +319,8 @@ class CallTypes(str, Enum): aretrieve_container = "aretrieve_container" delete_container = "delete_container" adelete_container = "adelete_container" + list_container_files = "list_container_files" + alist_container_files = "alist_container_files" acancel_fine_tuning_job = "acancel_fine_tuning_job" cancel_fine_tuning_job = "cancel_fine_tuning_job" diff --git a/tests/llm_translation/test_containers_api.py b/tests/llm_translation/test_containers_api.py new file mode 100644 index 0000000000..0226d7c44c --- /dev/null +++ b/tests/llm_translation/test_containers_api.py @@ -0,0 +1,119 @@ +""" +E2E Test for Container Files API. + +Tests the container files endpoints using LiteLLM SDK methods. +""" + +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.containers import ( + create_container, + delete_container, +) +from litellm.containers.endpoint_factory import ( + list_container_files, + retrieve_container_file, + retrieve_container_file_content, + delete_container_file, +) + + +@pytest.mark.skipif( + not os.getenv("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set" +) +def test_container_files_api(): + """ + Test container files API: list, retrieve, delete. + + Flow: + 1. Create a container + 2. List files (should be empty) + 3. Try retrieve file (should error - no files) + 4. Try delete file (should error - no files) + 5. Cleanup: delete container + """ + api_key = os.getenv("OPENAI_API_KEY") + + # 1. Create container + print("\n1. Creating container...") + container = create_container( + name=f"test-files-api-{int(time.time())}", + custom_llm_provider="openai", + api_key=api_key, + expires_after={"anchor": "last_active_at", "minutes": 5}, + ) + print(f" Created: {container.id}") + + try: + # 2. List files + print("2. Listing container files...") + files = list_container_files( + container_id=container.id, + custom_llm_provider="openai", + api_key=api_key, + ) + assert files.object == "list" + assert isinstance(files.data, list) + assert len(files.data) == 0 # New container has no files + print(f" Files found: {len(files.data)} ✓") + + # 3. Try retrieve non-existent file metadata (should raise error) + print("3. Testing retrieve_container_file (expect error)...") + try: + retrieve_container_file( + container_id=container.id, + file_id="cfile_nonexistent", + custom_llm_provider="openai", + api_key=api_key, + ) + assert False, "Should have raised error for non-existent file" + except Exception as e: + assert "not found" in str(e).lower() or "invalid" in str(e).lower() + print(f" Got expected error ✓") + + # 3b. Try retrieve non-existent file content (should raise error) + print("3b. Testing retrieve_container_file_content (expect error)...") + try: + retrieve_container_file_content( + container_id=container.id, + file_id="cfile_nonexistent", + custom_llm_provider="openai", + api_key=api_key, + ) + assert False, "Should have raised error for non-existent file content" + except Exception as e: + print(f" Got expected error ✓") + + # 4. Try delete non-existent file (should raise error) + print("4. Testing delete_container_file (expect error)...") + try: + delete_container_file( + container_id=container.id, + file_id="cfile_nonexistent", + custom_llm_provider="openai", + api_key=api_key, + ) + assert False, "Should have raised error for non-existent file" + except Exception as e: + # Delete returns 400 for non-existent files + print(f" Got expected error ✓") + + finally: + # 5. Cleanup + print("5. Deleting container...") + result = delete_container( + container_id=container.id, + custom_llm_provider="openai", + api_key=api_key, + ) + assert result.deleted is True + print(f" Deleted ✓") + + print("\nAll container files API tests passed! ✓") diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 18b5bb82dd..31cb87611a 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -51,6 +51,8 @@ import { fetchAvailableModels, ModelGroup } from "../llm_calls/fetch_models"; import { makeOpenAIImageEditsRequest } from "../llm_calls/image_edits"; import { makeOpenAIImageGenerationRequest } from "../llm_calls/image_generation"; import { makeOpenAIResponsesRequest } from "../llm_calls/responses_api"; +import CodeInterpreterOutput from "./CodeInterpreterOutput"; +import { useCodeInterpreter } from "./useCodeInterpreter"; import { Agent, fetchAvailableAgents } from "../llm_calls/fetch_agents"; import { makeA2AStreamMessageRequest } from "../llm_calls/a2a_send_message"; import A2AMetrics from "./A2AMetrics"; @@ -65,6 +67,7 @@ import { createDisplayMessage, createMultimodalMessage } from "./ResponsesImageU import { SearchResultsDisplay } from "./SearchResultsDisplay"; import SessionManagement from "./SessionManagement"; import { MessageType } from "./types"; +import CodeInterpreterTool from "./CodeInterpreterTool"; const { TextArea } = Input; const { Dragger } = Upload; @@ -197,6 +200,9 @@ const ChatUI: React.FC = ({ const [temperature, setTemperature] = useState(1.0); const [maxTokens, setMaxTokens] = useState(2048); const [useAdvancedParams, setUseAdvancedParams] = useState(false); + + // Code Interpreter state (using custom hook) + const codeInterpreter = useCodeInterpreter(); const chatEndRef = useRef(null); @@ -295,6 +301,7 @@ const ChatUI: React.FC = ({ sessionStorage.removeItem("responsesSessionId"); } sessionStorage.setItem("useApiSessionManagement", JSON.stringify(useApiSessionManagement)); + // Note: codeInterpreterEnabled and selectedContainerId are persisted by useCodeInterpreter hook }, [ apiKeySource, apiKey, @@ -736,6 +743,12 @@ const ChatUI: React.FC = ({ return; } + // Require model selection for Responses API + if (endpointType === EndpointType.RESPONSES && !selectedModel) { + NotificationsManager.fromBackend("Please select a model before sending a request"); + return; + } + if (!token || !userRole || !userID) { return; } @@ -809,6 +822,7 @@ const ChatUI: React.FC = ({ setChatHistory([...chatHistory, displayMessage]); setMCPEvents([]); // Clear previous MCP events for new conversation turn + codeInterpreter.clearResult(); // Clear previous code interpreter results setIsLoading(true); try { @@ -914,6 +928,8 @@ const ChatUI: React.FC = ({ useApiSessionManagement ? responsesSessionId : null, // Only pass session ID if API mode is enabled handleResponseId, // Pass callback to capture new response ID handleMCPEvent, // Pass MCP event handler + codeInterpreter.enabled, // Enable Code Interpreter tool + codeInterpreter.setResult, // Handle code interpreter output ); } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) { const apiChatHistory = [ @@ -1388,6 +1404,20 @@ const ChatUI: React.FC = ({ accessToken={accessToken || ""} /> + + {/* Code Interpreter Toggle - Only for Responses endpoint */} + {endpointType === EndpointType.RESPONSES && ( +
+ {}} + selectedModel={selectedModel || ""} + /> +
+ )} @@ -1468,6 +1498,19 @@ const ChatUI: React.FC = ({ )} + {/* Show Code Interpreter output for the last assistant message */} + {message.role === "assistant" && + index === chatHistory.length - 1 && + codeInterpreter.result && + endpointType === EndpointType.RESPONSES && ( + + )} +
= ({
)} + {/* Code Interpreter indicator and sample prompts when enabled */} + {endpointType === EndpointType.RESPONSES && codeInterpreter.enabled && ( +
+
+
+ {isLoading ? ( + <> + + Running Python code... + + ) : ( + <> + + Code Interpreter Active + + )} +
+ +
+ {/* Sample prompts - only show when not loading */} + {!isLoading && ( +
+ {[ + "Generate sample sales data CSV and create a chart", + "Create a PNG bar chart comparing AI gateway providers including LiteLLM", + "Generate a CSV of LLM pricing data and visualize it as a line chart", + ].map((prompt, idx) => ( + + ))} +
+ )} +
+ )} +
- {/* Left: paperclip icon */} -
+ {/* Left: attachment and code interpreter icons */} +
{endpointType === EndpointType.RESPONSES && !responsesUploadedImage && ( = ({ onRemoveImage={handleRemoveChatImage} /> )} + {/* Quick Code Interpreter toggle for Responses */} + {endpointType === EndpointType.RESPONSES && ( + + + + )}
{/* Middle: input field */} diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.tsx new file mode 100644 index 0000000000..5625d0cbf4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterOutput.tsx @@ -0,0 +1,224 @@ +import React, { useState, useEffect } from "react"; +import { Collapse, Spin } from "antd"; +import { + CodeOutlined, + DownloadOutlined, + FileImageOutlined, + FileTextOutlined, + LoadingOutlined, +} from "@ant-design/icons"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import { getProxyBaseUrl } from "@/components/networking"; + +interface ContainerFileCitation { + type: "container_file_citation"; + container_id: string; + file_id: string; + filename: string; + start_index: number; + end_index: number; +} + +interface CodeInterpreterOutputProps { + code?: string; + containerId?: string; + annotations?: ContainerFileCitation[]; + accessToken: string; +} + +const CodeInterpreterOutput: React.FC = ({ + code, + containerId, + annotations = [], + accessToken, +}) => { + const [imageUrls, setImageUrls] = useState>({}); + const [loadingImages, setLoadingImages] = useState>({}); + const proxyBaseUrl = getProxyBaseUrl(); + + // Fetch images from container files API + useEffect(() => { + const fetchImages = async () => { + for (const annotation of annotations) { + const isImage = annotation.filename?.toLowerCase().endsWith(".png") || + annotation.filename?.toLowerCase().endsWith(".jpg") || + annotation.filename?.toLowerCase().endsWith(".jpeg") || + annotation.filename?.toLowerCase().endsWith(".gif"); + + if (isImage && annotation.container_id && annotation.file_id) { + setLoadingImages(prev => ({ ...prev, [annotation.file_id]: true })); + + try { + // Fetch image content from container files API + const response = await fetch( + `${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + } + ); + + if (response.ok) { + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + setImageUrls(prev => ({ ...prev, [annotation.file_id]: url })); + } + } catch (error) { + console.error("Error fetching image:", error); + } finally { + setLoadingImages(prev => ({ ...prev, [annotation.file_id]: false })); + } + } + } + }; + + if (annotations.length > 0 && accessToken) { + fetchImages(); + } + + // Cleanup URLs on unmount + return () => { + Object.values(imageUrls).forEach(url => URL.revokeObjectURL(url)); + }; + }, [annotations, accessToken, proxyBaseUrl]); + + const handleDownload = async (annotation: ContainerFileCitation) => { + try { + const response = await fetch( + `${proxyBaseUrl}/v1/containers/${annotation.container_id}/files/${annotation.file_id}/content`, + { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + } + ); + + if (response.ok) { + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = annotation.filename || `file_${annotation.file_id}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + } catch (error) { + console.error("Error downloading file:", error); + } + }; + + // Separate images and other files + const imageAnnotations = annotations.filter(a => + a.filename?.toLowerCase().endsWith(".png") || + a.filename?.toLowerCase().endsWith(".jpg") || + a.filename?.toLowerCase().endsWith(".jpeg") || + a.filename?.toLowerCase().endsWith(".gif") + ); + + const fileAnnotations = annotations.filter(a => + !a.filename?.toLowerCase().endsWith(".png") && + !a.filename?.toLowerCase().endsWith(".jpg") && + !a.filename?.toLowerCase().endsWith(".jpeg") && + !a.filename?.toLowerCase().endsWith(".gif") + ); + + if (!code && annotations.length === 0) { + return null; + } + + return ( +
+ {/* Executed Code - Collapsible */} + {code && ( + + Python Code Executed + + ), + children: ( + + {code} + + ), + }, + ]} + /> + )} + + {/* Generated Images */} + {imageAnnotations.map((annotation) => ( +
+ {loadingImages[annotation.file_id] ? ( +
+ } /> + Loading image... +
+ ) : imageUrls[annotation.file_id] ? ( +
+ {annotation.filename +
+ + {annotation.filename} + + +
+
+ ) : ( +
+ Image not available +
+ )} +
+ ))} + + {/* Download Links for Other Files */} + {fileAnnotations.length > 0 && ( +
+ {fileAnnotations.map((annotation) => ( + + ))} +
+ )} +
+ ); +}; + +export default CodeInterpreterOutput; + diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterTool.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterTool.tsx new file mode 100644 index 0000000000..04711ede9f --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/CodeInterpreterTool.tsx @@ -0,0 +1,88 @@ +import React from "react"; +import { Switch, Tooltip, message } from "antd"; +import { CodeOutlined, InfoCircleOutlined, ExclamationCircleOutlined } from "@ant-design/icons"; +import { Text } from "@tremor/react"; + +interface CodeInterpreterToolProps { + accessToken: string; + enabled: boolean; + onEnabledChange: (enabled: boolean) => void; + selectedContainerId: string | null; + onContainerChange: (containerId: string | null) => void; + selectedModel: string; + disabled?: boolean; +} + +const GITHUB_FEATURE_REQUEST_URL = "https://github.com/BerriAI/litellm/issues/new?template=feature_request.yml"; + +const isOpenAIModel = (model: string): boolean => { + if (!model) return false; + const lowerModel = model.toLowerCase(); + return ( + lowerModel.startsWith("openai/") || + lowerModel.startsWith("gpt-") || + lowerModel.startsWith("o1") || + lowerModel.startsWith("o3") || + lowerModel.includes("openai") + ); +}; + +const CodeInterpreterTool: React.FC = ({ + enabled, + onEnabledChange, + selectedModel, + disabled = false, +}) => { + const isOpenAI = isOpenAIModel(selectedModel); + const isDisabled = disabled || !isOpenAI; + + const handleToggle = (checked: boolean) => { + if (checked && !isOpenAI) { + message.warning("Code Interpreter is only available for OpenAI models"); + return; + } + onEnabledChange(checked); + }; + + return ( +
+
+
+ + Code Interpreter + + + +
+ +
+ + {!isOpenAI && ( +
+
+ +
+ Code Interpreter is currently only supported for OpenAI models. + + Request support for other providers + +
+
+
+ )} +
+ ); +}; + +export default CodeInterpreterTool; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/useCodeInterpreter.ts b/ui/litellm-dashboard/src/components/playground/chat_ui/useCodeInterpreter.ts new file mode 100644 index 0000000000..78cd0aae3c --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/useCodeInterpreter.ts @@ -0,0 +1,57 @@ +/** + * Custom hook for managing Code Interpreter state. + * Container creation is handled automatically by OpenAI with container: { type: "auto" } + */ + +import { useState, useCallback } from "react"; +import { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler"; + +export interface UseCodeInterpreterReturn { + // State + enabled: boolean; + result: CodeInterpreterResult | null; + + // Actions + setEnabled: (enabled: boolean) => void; + setResult: (result: CodeInterpreterResult | null) => void; + clearResult: () => void; + toggle: () => void; +} + +export function useCodeInterpreter(): UseCodeInterpreterReturn { + const [enabled, setEnabledState] = useState(() => { + if (typeof window === "undefined") return false; + const saved = sessionStorage.getItem("codeInterpreterEnabled"); + return saved ? JSON.parse(saved) : false; + }); + + const [result, setResult] = useState(null); + + // Persist enabled state to session storage + const setEnabled = useCallback((value: boolean) => { + setEnabledState(value); + if (typeof window !== "undefined") { + sessionStorage.setItem("codeInterpreterEnabled", JSON.stringify(value)); + } + }, []); + + const clearResult = useCallback(() => { + setResult(null); + }, []); + + const toggle = useCallback(() => { + setEnabled(!enabled); + }, [enabled, setEnabled]); + + return { + enabled, + result, + setEnabled, + setResult, + clearResult, + toggle, + }; +} + +// Re-export the type for convenience +export type { CodeInterpreterResult } from "../llm_calls/code_interpreter_handler"; diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/code_interpreter_handler.ts b/ui/litellm-dashboard/src/components/playground/llm_calls/code_interpreter_handler.ts new file mode 100644 index 0000000000..7132b55434 --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/code_interpreter_handler.ts @@ -0,0 +1,83 @@ +/** + * Code Interpreter event handling for the Responses API. + */ + +export interface CodeInterpreterResult { + code: string; + containerId: string; + annotations: Array<{ + type: "container_file_citation"; + container_id: string; + file_id: string; + filename: string; + start_index: number; + end_index: number; + }>; +} + +export interface CodeInterpreterState { + code: string; + containerId: string; +} + +/** + * Handle code interpreter call completed event. + * Extracts code and container ID from the event. + */ +export function handleCodeInterpreterCall( + event: any, + state: CodeInterpreterState +): CodeInterpreterState { + if (event.type === "response.output_item.done" && event.item?.type === "code_interpreter_call") { + console.log("Code interpreter call completed:", event.item); + return { + code: event.item.code || "", + containerId: event.item.container_id || "", + }; + } + return state; +} + +/** + * Handle code interpreter output with file annotations. + * Calls the callback if file annotations are present. + */ +export function handleCodeInterpreterOutput( + event: any, + state: CodeInterpreterState, + onCodeInterpreterResult?: (result: CodeInterpreterResult) => void +): void { + if ( + event.type === "response.output_item.done" && + event.item?.type === "message" && + event.item?.content && + onCodeInterpreterResult + ) { + const content = event.item.content; + for (const part of content) { + if (part.type === "output_text" && part.annotations) { + const fileAnnotations = part.annotations.filter( + (a: any) => a.type === "container_file_citation" + ); + if (fileAnnotations.length > 0 || state.code) { + onCodeInterpreterResult({ + code: state.code, + containerId: state.containerId, + annotations: fileAnnotations, + }); + } + } + } + } +} + +/** + * Check if code interpreter is being used based on event type. + */ +export function isCodeInterpreterEvent(event: any): boolean { + return ( + event.type === "response.output_item.done" && + event.item?.type === "code_interpreter_call" + ); +} + diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx index 46b0621a0b..8488f012e7 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/responses_api.tsx @@ -4,6 +4,14 @@ import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getProxyBaseUrl } from "@/components/networking"; import NotificationManager from "@/components/molecules/notifications_manager"; import { MCPEvent } from "../chat_ui/MCPEventsDisplay"; +import { + CodeInterpreterResult, + CodeInterpreterState, + handleCodeInterpreterCall, + handleCodeInterpreterOutput, +} from "./code_interpreter_handler"; + +export type { CodeInterpreterResult } from "./code_interpreter_handler"; export async function makeOpenAIResponsesRequest( messages: MessageType[], @@ -22,11 +30,17 @@ export async function makeOpenAIResponsesRequest( previousResponseId?: string | null, onResponseId?: (responseId: string) => void, onMCPEvent?: (event: MCPEvent) => void, + codeInterpreterEnabled?: boolean, + onCodeInterpreterResult?: (result: CodeInterpreterResult) => void, ) { if (!accessToken) { throw new Error("Virtual Key is required"); } + if (!selectedModel || selectedModel.trim() === "") { + throw new Error("Model is required. Please select a model before sending a request."); + } + // Base URL should be the current base_url const isLocal = process.env.NODE_ENV === "development"; if (isLocal !== true) { @@ -69,19 +83,27 @@ export async function makeOpenAIResponsesRequest( }; }); - // Format MCP tools if selected - const tools = - selectedMCPTools && selectedMCPTools.length > 0 - ? [ - { - type: "mcp", - server_label: "litellm", - server_url: `litellm_proxy/mcp`, - require_approval: "never", - allowed_tools: selectedMCPTools, - }, - ] - : undefined; + // Build tools array + const tools: any[] = []; + + // Add MCP tools if selected + if (selectedMCPTools && selectedMCPTools.length > 0) { + tools.push({ + type: "mcp", + server_label: "litellm", + server_url: `litellm_proxy/mcp`, + require_approval: "never", + allowed_tools: selectedMCPTools, + }); + } + + // Add code_interpreter tool if enabled (OpenAI auto-creates container) + if (codeInterpreterEnabled) { + tools.push({ + type: "code_interpreter", + container: { type: "auto" }, + }); + } // Create request to OpenAI responses API // Use 'any' type to avoid TypeScript issues with the experimental API @@ -94,12 +116,13 @@ export async function makeOpenAIResponsesRequest( ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), ...(vector_store_ids ? { vector_store_ids } : {}), ...(guardrails ? { guardrails } : {}), - ...(tools ? { tools, tool_choice: "required" } : {}), + ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), }, { signal }, ); let mcpToolUsed = ""; + let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" }; for await (const event of response) { console.log("Response event:", event); @@ -137,6 +160,10 @@ export async function makeOpenAIResponsesRequest( console.log("MCP tool used:", mcpToolUsed); } + // Handle code interpreter events + codeInterpreterState = handleCodeInterpreterCall(event, codeInterpreterState); + handleCodeInterpreterOutput(event, codeInterpreterState, onCodeInterpreterResult); + // Handle output text delta // 1) drop any "role" streams if (event.type === "response.role.delta") {