tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-02 08:21:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(containers): Azure routing, managed container IDs, delete response parsing (#25287)

Browse Source 
* feat(containers): Azure container routing, managed IDs, and delete response wire format

- Add AzureContainerConfig and safe URL joining for paths with api-version query
- Encode/decode managed container IDs in responses, streaming, and proxy handlers
- Accept OpenAI delete response object literal container.file.deleted
- Tests for Azure URL regression and DeleteContainerFileResponse parsing

Made-with: Cursor

* fix(responses): gate response id update on parsed_chunk having response

Delta stream events do not include a response body; Mock-based tests
(and any truthy synthetic .response on transforms) must not trigger
_update_responses_api_response_id_with_model_id. Fixes
test_stop_async_iteration_not_logged_as_failure (TypeError: Mock not iterable).

Made-with: Cursor

* feat(containers): encode container IDs in SDK responses for routing

- Add ContainerRequestUtils.encode_container_id_in_response utility
- Encode container_id in create/retrieve/delete responses (SDK path)
- Fix streaming iterator: gate response ID update on parsed_chunk key
- Follows responses API pattern (encode after handler, not in handler)

Made-with: Cursor

* fix(containers): module-level imports and managed cntr_ ID encoding

- Move ResponsesAPIRequestUtils imports to module scope (utils, main, handler_factory).
- Serialize absent model_id as empty segment instead of literal None; decode empty
  and legacy "None" segments as missing for router affinity.
- Add unit tests for build/decode round-trip and legacy IDs.

Made-with: Cursor

* fix(containers): decode managed IDs in endpoint_factory SDK path

- Add decode_managed_container_id_for_request in containers/utils and reuse from main.
- Strip LiteLLM cntr_ wrappers before generic_container_handler (64-char API limit).
- Resolve provider for logging/errors; add unit test for decode helper.
- Use resolved_custom_llm_provider after decode for mypy-safe provider typing.

Made-with: Cursor

* Fix p1 concern

* Fix p1 concern
This commit is contained in:
Sameer Kankute
2026-04-11 09:21:01 -07:00
committed by GitHub
parent 40d8a25df9
commit d03ecedba1
17 changed files with 1381 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/containers/endpoint_factory.py
+20 -6
View File
@@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Type
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.containers.utils import decode_managed_container_id_for_request
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.llms.custom_httpx.container_handler import generic_container_handler
@@ -53,7 +54,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
@client
def endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@@ -61,6 +62,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
):
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj")
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@@ -76,15 +78,27 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
# Get provider config
litellm_params = GenericLiteLLMParams(**kwargs)
# Strip LiteLLM-managed container IDs before calling the provider API
# (OpenAI enforces max length 64 on container_id).
if "container_id" in kwargs and isinstance(kwargs["container_id"], str):
(
kwargs["container_id"],
resolved_custom_llm_provider,
litellm_params,
) = decode_managed_container_id_for_request(
container_id=kwargs["container_id"],
custom_llm_provider=resolved_custom_llm_provider,
litellm_params=litellm_params,
)
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for: {custom_llm_provider}"
f"Container provider config not found for: {resolved_custom_llm_provider}"
)
# Build optional params for logging
@@ -96,7 +110,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
model="",
optional_params=optional_params,
litellm_params={"litellm_call_id": litellm_call_id},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Use generic handler
@@ -115,7 +129,7 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@@ -133,7 +147,7 @@ def create_async_endpoint_function(
@client
async def async_endpoint_func(
timeout: int = 600,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
litellm/containers/main.py
+154 -48
View File
@@ -6,7 +6,10 @@ from typing import Any, Coroutine, Dict, List, Literal, Optional, Union, overloa
import litellm
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.containers.utils import ContainerRequestUtils
from litellm.containers.utils import (
ContainerRequestUtils,
decode_managed_container_id_for_request,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.main import base_llm_http_handler
@@ -48,7 +51,7 @@ async def acreate_container(
file_ids: Optional[List[str]] = None,
timeout=600, # default to 10 minutes
# LiteLLM specific params,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -122,7 +125,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
acreate_container: Literal[True],
**kwargs,
@@ -139,7 +142,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
acreate_container: Literal[False] = False,
**kwargs,
@@ -158,7 +161,7 @@ def create_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -247,7 +250,7 @@ def create_container(
# Set the correct call type for container creation
litellm_logging_obj.call_type = CallTypes.create_container.value
return base_llm_http_handler.container_create_handler(
container_obj = base_llm_http_handler.container_create_handler(
name=name,
container_create_request_params=container_create_request_params,
container_provider_config=container_provider_config,
@@ -257,6 +260,17 @@ def create_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
if isinstance(container_obj, ContainerObject):
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return container_obj
except Exception as e:
raise litellm.exception_type(
@@ -275,7 +289,7 @@ async def alist_containers(
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -348,7 +362,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_containers: Literal[True],
**kwargs,
@@ -365,7 +379,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_containers: Literal[False] = False,
**kwargs,
@@ -384,7 +398,7 @@ def list_containers(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -481,7 +495,7 @@ def list_containers(
async def aretrieve_container(
container_id: str,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -548,7 +562,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aretrieve_container: Literal[True],
**kwargs,
@@ -563,7 +577,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aretrieve_container: Literal[False] = False,
**kwargs,
@@ -580,7 +594,7 @@ def retrieve_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -594,6 +608,7 @@ def retrieve_container(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@@ -615,16 +630,28 @@ def retrieve_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# True when input was a LiteLLM-managed ID (any length); needed to re-encode output for routing affinity
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@@ -635,14 +662,14 @@ def retrieve_container(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.retrieve_container.value
return base_llm_http_handler.container_retrieve_handler(
container_id=container_id,
container_obj = base_llm_http_handler.container_retrieve_handler(
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@@ -651,11 +678,33 @@ def retrieve_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(container_obj, ContainerObject):
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return container_obj
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@@ -667,7 +716,7 @@ def retrieve_container(
async def adelete_container(
container_id: str,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -734,7 +783,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
adelete_container: Literal[True],
**kwargs,
@@ -749,7 +798,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
adelete_container: Literal[False] = False,
**kwargs,
@@ -766,7 +815,7 @@ def delete_container(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
# The extra values given here take precedence over values defined on the client or passed to this method.
extra_headers: Optional[Dict[str, Any]] = None,
@@ -780,6 +829,7 @@ def delete_container(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@@ -801,16 +851,28 @@ def delete_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# True when input was a LiteLLM-managed ID (any length); needed to re-encode output for routing affinity
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@@ -821,14 +883,14 @@ def delete_container(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.delete_container.value
return base_llm_http_handler.container_delete_handler(
container_id=container_id,
delete_result = base_llm_http_handler.container_delete_handler(
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@@ -837,11 +899,33 @@ def delete_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id in response with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(delete_result, DeleteContainerResult):
# If input was encoded, use model_id from decoded params
litellm_metadata = kwargs.get("litellm_metadata", {})
if was_encoded and litellm_params.get("model_id"):
# Inject model_id from decoded container_id into litellm_metadata
if not litellm_metadata:
litellm_metadata = {}
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
delete_result = ContainerRequestUtils.encode_container_id_in_response(
response_obj=delete_result,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return delete_result
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@@ -856,7 +940,7 @@ async def alist_container_files(
limit: Optional[int] = None,
order: Optional[str] = None,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@@ -930,7 +1014,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_container_files: Literal[True],
**kwargs,
@@ -948,7 +1032,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
alist_container_files: Literal[False] = False,
**kwargs,
@@ -968,7 +1052,7 @@ def list_container_files(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@@ -980,6 +1064,7 @@ def list_container_files(
"""
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@@ -1001,16 +1086,26 @@ def list_container_files(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@@ -1026,14 +1121,14 @@ def list_container_files(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
litellm_logging_obj.call_type = CallTypes.list_container_files.value
return base_llm_http_handler.container_file_list_handler(
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
container_provider_config=container_provider_config,
litellm_params=litellm_params,
logging_obj=litellm_logging_obj,
@@ -1049,7 +1144,7 @@ def list_container_files(
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
@@ -1062,7 +1157,7 @@ async def aupload_container_file(
container_id: str,
file: FileTypes,
timeout=600, # default to 10 minutes
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@@ -1151,7 +1246,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aupload_container_file: Literal[True],
**kwargs,
@@ -1167,7 +1262,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
*,
aupload_container_file: Literal[False] = False,
**kwargs,
@@ -1185,7 +1280,7 @@ def upload_container_file(
api_key: Optional[str] = None,
api_base: Optional[str] = None,
api_version: Optional[str] = None,
custom_llm_provider: Literal["openai"] = "openai",
custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
extra_headers: Optional[Dict[str, Any]] = None,
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
@@ -1226,6 +1321,7 @@ def upload_container_file(
local_vars = locals()
try:
resolved_custom_llm_provider: str = custom_llm_provider
litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id")
_is_async = kwargs.pop("async_call", False) is True
@@ -1247,16 +1343,26 @@ def upload_container_file(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
container_id=container_id,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
if container_provider_config is None:
raise ValueError(
f"Container provider config not found for provider: {custom_llm_provider}"
f"Container provider config not found for provider: {resolved_custom_llm_provider}"
)
# Pre Call logging
@@ -1267,7 +1373,7 @@ def upload_container_file(
litellm_params={
"litellm_call_id": litellm_call_id,
},
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
)
# Set the correct call type
@@ -1282,14 +1388,14 @@ def upload_container_file(
extra_query=extra_query,
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
file=file,
)
except Exception as e:
raise litellm.exception_type(
model="",
custom_llm_provider=custom_llm_provider,
custom_llm_provider=resolved_custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
extra_kwargs=kwargs,
litellm/containers/utils.py
+92 -1
View File
@@ -1,10 +1,38 @@
from typing import Dict
from typing import Any, Dict, Optional, TypeVar
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerListOptionalRequestParams,
)
from litellm.types.router import GenericLiteLLMParams
def decode_managed_container_id_for_request(
container_id: str,
custom_llm_provider: str,
litellm_params: GenericLiteLLMParams,
) -> tuple[str, str, GenericLiteLLMParams]:
"""Decode a LiteLLM-managed container ID for upstream API calls.
Returns:
(original_container_id, resolved_provider, updated_litellm_params)
"""
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
decoded_model_id = decoded.get("model_id")
if decoded_model_id and not litellm_params.get("model_id"):
litellm_params["model_id"] = decoded_model_id
return original_container_id, custom_llm_provider, litellm_params
T = TypeVar("T")
class ContainerRequestUtils:
@@ -68,3 +96,66 @@ class ContainerRequestUtils:
container_list_optional_params[param] = passed_params[param] # type: ignore
return container_list_optional_params
@staticmethod
def encode_container_id_in_response(
response_obj: T,
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
) -> T:
"""
Encode container_id in response object with provider/model metadata for routing.
This mirrors the responses API pattern where response IDs are encoded with
routing metadata so follow-up calls can route to the correct provider.
Encodes when:
1. litellm_metadata contains model_info.id (indicating router/proxy usage), OR
2. extra_body contains target_model_names (indicating model-specific routing)
Direct SDK calls with explicit custom_llm_provider and no routing hints return raw IDs.
Args:
response_obj: Response object with an `id` attribute (ContainerObject, DeleteContainerResult, etc.)
custom_llm_provider: Provider name (e.g., "azure", "openai")
litellm_metadata: Optional litellm_metadata dict that may contain model_info.id
extra_body: Optional extra_body dict that may contain target_model_names
Returns:
The same response object with encoded container_id (if routing metadata present)
"""
# Extract model_id from litellm_metadata
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Check if we should encode based on routing metadata
should_encode = False
# Case 1: Router/proxy usage (model_id from router)
if model_id is not None:
should_encode = True
# Case 2: target_model_names in extra_body (model-specific routing)
if extra_body and "target_model_names" in extra_body:
should_encode = True
# Extract model_id from target_model_names if not already set
if model_id is None:
target_models = extra_body["target_model_names"]
# Use first model as model_id for encoding
if isinstance(target_models, str):
model_id = target_models.split(",")[0].strip()
elif isinstance(target_models, list) and len(target_models) > 0:
model_id = str(target_models[0]).strip()
# Only encode if we have routing metadata
if should_encode and response_obj and hasattr(response_obj, "id"):
encoded_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
container_id=response_obj.id,
)
response_obj.id = encoded_id
return response_obj
litellm/llms/azure/containers/__init__.py
View File
litellm/llms/azure/containers/transformation.py
+48
View File
@@ -0,0 +1,48 @@
from typing import Optional
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.types.router import GenericLiteLLMParams
class AzureContainerConfig(OpenAIContainerConfig):
"""
Configuration class for Azure OpenAI container API.
Inherits request/response transformations from OpenAIContainerConfig since
Azure's container API is wire-compatible with OpenAI's. Only overrides
authentication (api-key header) and URL construction (openai/v1/containers path).
Azure container API reference:
https://learn.microsoft.com/en-us/azure/foundry/openai/latest#containers
"""
def validate_environment(
self,
headers: dict,
api_key: Optional[str] = None,
) -> dict:
return BaseAzureLLM._base_validate_azure_environment(
headers=headers,
litellm_params=GenericLiteLLMParams(api_key=api_key),
)
def get_complete_url(
self,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Build the Azure container endpoint URL.
Azure container API uses the path:
{endpoint}/openai/v1/containers
when api_version is 'v1', 'latest', or 'preview'; otherwise:
{endpoint}/openai/containers
"""
return BaseAzureLLM._get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
route="/openai/containers",
default_api_version="v1",
)
litellm/llms/custom_httpx/container_handler.py
+16 -5
View File
@@ -61,18 +61,29 @@ def _build_url(
) -> str:
"""Build the full URL by substituting path parameters.
The api_base from get_complete_url already includes /containers,
so we need to strip that prefix from the path_template.
The api_base from get_complete_url already includes /containers and may include
query parameters. We need to parse the URL, append the path, then preserve the
query parameters.
"""
# api_base ends with /containers, path_template starts with /containers
# So we need to strip /containers from the path
if path_template.startswith("/containers"):
path_template = path_template[len("/containers") :]
url = f"{api_base.rstrip('/')}{path_template}"
# Substitute path parameters
for param, value in path_params.items():
url = url.replace(f"{{{param}}}", value)
return url
path_template = path_template.replace(f"{{{param}}}", value)
# Parse the api_base to extract existing query params
parsed_base = httpx.URL(api_base)
# Append the path to the existing path (before query params)
new_path = f"{parsed_base.path.rstrip('/')}{path_template}"
# Rebuild URL with new path, preserving query params
final_url = parsed_base.copy_with(path=new_path)
return str(final_url)
def _build_query_params(
litellm/llms/openai/containers/transformation.py
+7 -4
View File
@@ -17,6 +17,7 @@ from litellm.types.containers.main import (
from litellm.types.router import GenericLiteLLMParams
from ...base_llm.containers.transformation import BaseContainerConfig
from .utils import join_container_api_base_path
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -197,7 +198,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
) -> Tuple[str, Dict]:
"""Transform the OpenAI container retrieve request."""
# For container retrieve, we just need to construct the URL
url = f"{api_base.rstrip('/')}/{container_id}"
url = join_container_api_base_path(api_base, f"/{container_id}")
# No additional data needed for GET request
data: Dict[str, Any] = {}
@@ -229,7 +230,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
- DELETE /v1/containers/{container_id}
"""
# Construct the URL for container delete
url = f"{api_base.rstrip('/')}/{container_id}"
url = join_container_api_base_path(api_base, f"/{container_id}")
# No data needed for DELETE request
data: Dict[str, Any] = {}
@@ -266,7 +267,7 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files
"""
# Construct the URL for container files
url = f"{api_base.rstrip('/')}/{container_id}/files"
url = join_container_api_base_path(api_base, f"/{container_id}/files")
# Prepare query parameters
params: Dict[str, Any] = {}
@@ -310,7 +311,9 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files/{file_id}/content
"""
# Construct the URL for container file content
url = f"{api_base.rstrip('/')}/{container_id}/files/{file_id}/content"
url = join_container_api_base_path(
api_base, f"/{container_id}/files/{file_id}/content"
)
# No query parameters needed
params: Dict[str, Any] = {}
litellm/llms/openai/containers/utils.py
+18
View File
@@ -0,0 +1,18 @@
"""Shared helpers for OpenAI-compatible container API URL construction."""
import httpx
def join_container_api_base_path(api_base: str, path_suffix: str) -> str:
"""Append ``path_suffix`` to the path of ``api_base``, keeping the query string last.
Azure (and some bases) pass ``api_base`` like
``https://host/openai/v1/containers?api-version=v1``. Naive string concat would
produce ``...?api-version=v1/cntr_...`` which is invalid; this uses ``httpx.URL``
so the result is ``.../containers/cntr_.../files?api-version=v1``.
"""
if not path_suffix.startswith("/"):
path_suffix = f"/{path_suffix}"
parsed = httpx.URL(api_base)
new_path = f"{parsed.path.rstrip('/')}{path_suffix}"
return str(parsed.copy_with(path=new_path))
litellm/proxy/container_endpoints/handler_factory.py
+48 -10
View File
@@ -19,6 +19,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
get_custom_llm_provider_from_request_headers,
get_custom_llm_provider_from_request_query,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
def _load_endpoints_config() -> Dict:
@@ -40,10 +41,13 @@ def _get_container_provider_config(custom_llm_provider: str):
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
return OpenAIContainerConfig()
else:
raise ValueError(
f"Container API not supported for provider: {custom_llm_provider}"
)
elif custom_llm_provider in ("azure", "azure_text"):
from litellm.llms.azure.containers.transformation import AzureContainerConfig
return AzureContainerConfig()
raise ValueError(
f"Container API not supported for provider: {custom_llm_provider}"
)
def _create_handler_for_path_params(
@@ -171,12 +175,21 @@ async def _process_binary_request(
or "openai"
)
# Get the provider config
container_provider_config = _get_container_provider_config(custom_llm_provider)
# Build litellm_params - credentials are resolved by provider config from env
litellm_params = GenericLiteLLMParams()
# Decode container ID and extract provider info
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
# Get the provider config
container_provider_config = _get_container_provider_config(custom_llm_provider)
# Create logging object
logging_obj = Logging(
model="container-file-content",
@@ -193,7 +206,7 @@ async def _process_binary_request(
try:
content = await handler.async_container_file_content_handler(
container_id=container_id,
container_id=original_container_id, # Use decoded original ID
file_id=file_id,
container_provider_config=container_provider_config,
litellm_params=litellm_params,
@@ -267,13 +280,22 @@ async def _process_multipart_upload_request(
if isinstance(file_list, list) and len(file_list) > 0:
data["file"] = file_list[0]
data["container_id"] = container_id
custom_llm_provider = (
get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
# Decode container ID and extract provider info
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
original_container_id = decoded.get("response_id", container_id)
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
data["container_id"] = original_container_id # Use decoded original ID
data["custom_llm_provider"] = custom_llm_provider
processor = ProxyBaseLLMRequestProcessing(data=data)
@@ -338,6 +360,22 @@ async def _process_request(
or get_custom_llm_provider_from_request_query(request=request)
or "openai"
)
# Decode container_id if present in path_params
if "container_id" in path_params:
decoded = ResponsesAPIRequestUtils._decode_container_id(
path_params["container_id"]
)
original_container_id = decoded.get("response_id", path_params["container_id"])
# If container ID has encoded provider info and user didn't explicitly set provider, use it
decoded_provider = decoded.get("custom_llm_provider")
if decoded_provider and custom_llm_provider == "openai":
custom_llm_provider = decoded_provider
# Update path_params with decoded original ID
data["container_id"] = original_container_id
data["custom_llm_provider"] = custom_llm_provider
processor = ProxyBaseLLMRequestProcessing(data=data)
litellm/responses/streaming_iterator.py
+57 -8
View File
@@ -130,15 +130,64 @@ class BaseResponsesAPIStreamingIterator:
)
)
# if "response" in parsed_chunk, then encode litellm specific information like custom_llm_provider
response_object = getattr(openai_responses_api_chunk, "response", None)
if response_object:
response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id(
responses_api_response=response_object,
litellm_metadata=self.litellm_metadata,
custom_llm_provider=self.custom_llm_provider,
# Only when the SSE JSON carries a response body (delta events do not).
# Using getattr(..., "response") alone is unsafe with Mocks: they synthesize a
# truthy child Mock for any attribute, which breaks tests and is wrong on stream.
if "response" in parsed_chunk:
response_object = getattr(
openai_responses_api_chunk, "response", None
)
setattr(openai_responses_api_chunk, "response", response)
if response_object is not None:
response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id(
responses_api_response=response_object,
litellm_metadata=self.litellm_metadata,
custom_llm_provider=self.custom_llm_provider,
)
setattr(openai_responses_api_chunk, "response", response)
# Encode container_id on streaming events so proxy/UI follow-ups route correctly
_event_type = getattr(openai_responses_api_chunk, "type", None)
_stream_model_id = (
self.litellm_metadata.get("model_info", {}).get("id")
if self.litellm_metadata
else None
)
if _event_type in (
ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
):
_item = getattr(openai_responses_api_chunk, "item", None)
if _item is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=_item,
custom_llm_provider=self.custom_llm_provider,
model_id=_stream_model_id,
)
elif _event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED:
_annotation = getattr(
openai_responses_api_chunk, "annotation", None
)
if _annotation is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=_annotation,
custom_llm_provider=self.custom_llm_provider,
model_id=_stream_model_id,
)
elif _event_type == ResponsesAPIStreamEvents.CONTENT_PART_DONE:
_part = getattr(openai_responses_api_chunk, "part", None)
if _part is not None:
if isinstance(_part, dict):
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
_part.get("annotations"),
self.custom_llm_provider,
_stream_model_id,
)
else:
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
getattr(_part, "annotations", None),
self.custom_llm_provider,
_stream_model_id,
)
# Wrap encrypted_content in streaming events (output_item.added, output_item.done)
if self.litellm_metadata and self.litellm_metadata.get(
litellm/responses/utils.py
+249
View File
@@ -1,4 +1,5 @@
import base64
import re
from typing import (
Any,
Dict,
@@ -226,6 +227,15 @@ class ResponsesAPIRequestUtils:
)
)
# Encode container IDs in the response output
responses_api_response = (
ResponsesAPIRequestUtils._update_container_ids_in_response(
responses_api_response=responses_api_response,
custom_llm_provider=custom_llm_provider,
litellm_metadata=litellm_metadata,
)
)
return responses_api_response
@staticmethod
@@ -522,6 +532,245 @@ class ResponsesAPIRequestUtils:
)
return decoded_response_id.get("response_id", previous_response_id)
@staticmethod
def _build_container_id(
custom_llm_provider: Optional[str],
model_id: Optional[str],
container_id: str,
) -> str:
"""Build a managed container ID with provider and model info encoded.
Format: cntr_{base64("litellm:custom_llm_provider:{provider};model_id:{model};container_id:{original}")}
"""
# Avoid serializing Python None as the literal string "None" (breaks router affinity).
provider_part = "" if custom_llm_provider is None else custom_llm_provider
model_part = "" if model_id is None else model_id
assembled_id = f"litellm:custom_llm_provider:{provider_part};model_id:{model_part};container_id:{container_id}"
base64_encoded_id = base64.b64encode(assembled_id.encode("utf-8")).decode("utf-8")
return f"cntr_{base64_encoded_id}"
@staticmethod
def _decode_container_id(container_id: str) -> DecodedResponseId:
"""Decode a managed container ID to extract provider, model, and original container ID.
Returns:
DecodedResponseId with custom_llm_provider, model_id, and response_id (original container_id)
"""
try:
# If it doesn't start with cntr_, it's not a managed ID
if not container_id.startswith("cntr_"):
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
# Remove prefix and decode
cleaned_id = container_id.replace("cntr_", "")
decoded_id = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8")
# Parse components using regex to handle semicolons in the container_id
if not decoded_id.startswith("litellm:"):
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
# Use regex to extract the three parts, allowing semicolons in container_id
# Format: litellm:custom_llm_provider:{provider};model_id:{model};container_id:{container}
# * for provider/model allows empty segments (missing router model_id).
pattern = r"^litellm:custom_llm_provider:([^;]*);model_id:([^;]*);container_id:(.+)$"
match = re.match(pattern, decoded_id)
if not match:
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
raw_provider = match.group(1)
raw_model_id = match.group(2)
custom_llm_provider = (
None if raw_provider in ("", "None") else raw_provider
)
model_id = None if raw_model_id in ("", "None") else raw_model_id
original_container_id = match.group(3)
return DecodedResponseId(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
response_id=original_container_id,
)
except Exception as e:
verbose_logger.debug(f"Error decoding container_id '{container_id}': {e}")
return DecodedResponseId(
custom_llm_provider=None,
model_id=None,
response_id=container_id,
)
@staticmethod
def decode_container_id_to_original(container_id: str) -> str:
"""Decode a managed container ID to get the original provider-issued ID.
This is used when making upstream API calls - we need to send the original
container ID that the provider issued, not our encoded version.
"""
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
return decoded.get("response_id", container_id)
@staticmethod
def _encode_container_ids_in_annotations(
annotations: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Encode ``container_id`` on each annotation (e.g. ``container_file_citation``)."""
if not annotations or not isinstance(annotations, list):
return
for ann in annotations:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
ann,
custom_llm_provider,
model_id,
)
@staticmethod
def _encode_container_ids_in_message_content(
content: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Walk message ``content`` parts and encode citation ``container_id`` values."""
if not content:
return
if isinstance(content, list):
for part in content:
if isinstance(part, dict):
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
part.get("annotations"),
custom_llm_provider,
model_id,
)
else:
ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
getattr(part, "annotations", None),
custom_llm_provider,
model_id,
)
@staticmethod
def _encode_container_id_on_output_item(
item: Any,
custom_llm_provider: Optional[str],
model_id: Optional[str],
) -> None:
"""Mutate one output item (dict or object): wrap raw ``container_id`` as LiteLLM-managed.
Handles top-level ``container_id`` and nested ``code_interpreter_call.container_id``
(some wire payloads nest the tool call). Used by non-streaming responses and by
streaming ``response.output_item.*`` events so UIs see managed IDs incrementally.
For ``message`` items, also encodes ``container_id`` inside
``content[].annotations`` (``container_file_citation``), which is what clients use
to fetch generated files.
"""
if item is None:
return
def _maybe_encode(container_id: str) -> Optional[str]:
decoded = ResponsesAPIRequestUtils._decode_container_id(container_id)
if decoded.get("custom_llm_provider") is not None:
return None
return ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider=custom_llm_provider,
model_id=model_id,
container_id=container_id,
)
if isinstance(item, dict):
cid = item.get("container_id")
if isinstance(cid, str):
enc = _maybe_encode(cid)
if enc is not None:
item["container_id"] = enc
nested = item.get("code_interpreter_call")
if isinstance(nested, dict):
nc = nested.get("container_id")
if isinstance(nc, str):
enc = _maybe_encode(nc)
if enc is not None:
nested["container_id"] = enc
if item.get("type") == "message":
ResponsesAPIRequestUtils._encode_container_ids_in_message_content(
item.get("content"),
custom_llm_provider,
model_id,
)
return
cid_attr = getattr(item, "container_id", None)
if isinstance(cid_attr, str):
enc = _maybe_encode(cid_attr)
if enc is not None:
try:
setattr(item, "container_id", enc)
except Exception:
verbose_logger.debug(
"Could not set container_id on streaming output item",
exc_info=True,
)
nested_obj = getattr(item, "code_interpreter_call", None)
if nested_obj is not None:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
nested_obj,
custom_llm_provider,
model_id,
)
if getattr(item, "type", None) == "message":
ResponsesAPIRequestUtils._encode_container_ids_in_message_content(
getattr(item, "content", None),
custom_llm_provider,
model_id,
)
@staticmethod
def _update_container_ids_in_response(
responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]],
custom_llm_provider: Optional[str],
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Union[ResponsesAPIResponse, Dict[str, Any]]:
"""Encode container IDs in the response output with provider/model info.
This walks through all output items and encodes any container_id fields
so that follow-up container API calls can auto-route to the correct provider.
"""
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Get the output list
if isinstance(responses_api_response, dict):
output = responses_api_response.get("output", [])
else:
output = getattr(responses_api_response, "output", [])
if not output:
return responses_api_response
for item in output:
ResponsesAPIRequestUtils._encode_container_id_on_output_item(
item=item,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
)
return responses_api_response
@staticmethod
def convert_text_format_to_text_param(
text_format: Optional[Union[Type["BaseModel"], dict]],
litellm/types/containers/main.py
+2 -1
View File
@@ -187,7 +187,8 @@ class DeleteContainerFileResponse(BaseModel):
"""Response object for delete container file request."""
id: str
object: Literal["container_file.deleted"]
# OpenAI / Azure wire format uses dots; keep underscore variant for compatibility.
object: Literal["container.file.deleted", "container_file.deleted"]
deleted: bool
def __contains__(self, key):
litellm/utils.py
+6
View File
@@ -8955,6 +8955,12 @@ class ProviderConfigManager:
)
return OpenAIContainerConfig()
if provider in (LlmProviders.AZURE, LlmProviders.AZURE_TEXT):
from litellm.llms.azure.containers.transformation import (
AzureContainerConfig,
)
return AzureContainerConfig()
return None
@staticmethod
tests/test_litellm/containers/test_azure_container_transformation.py
+519
View File
@@ -0,0 +1,519 @@
import os
import sys
from unittest.mock import MagicMock
from urllib.parse import parse_qs, urlparse
import httpx
import pytest
sys.path.insert(0, os.path.abspath("../../../"))
import litellm
from litellm.llms.azure.containers.transformation import AzureContainerConfig
from litellm.llms.base_llm.containers.transformation import BaseContainerConfig
from litellm.types.containers.main import (
ContainerFileListResponse,
ContainerListResponse,
ContainerObject,
DeleteContainerResult,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
class TestAzureContainerConfig:
"""Test suite for Azure container transformation functionality."""
def setup_method(self):
self.config = AzureContainerConfig()
self.logging_obj = LiteLLMLogging(
model="",
messages=[],
stream=False,
call_type="create_container",
start_time=None,
litellm_call_id="test_call_id",
function_id="test_function_id",
)
def test_inherits_base_container_config(self):
assert isinstance(self.config, BaseContainerConfig)
def test_get_supported_openai_params(self):
supported_params = self.config.get_supported_openai_params()
assert "name" in supported_params
assert "expires_after" in supported_params
assert "file_ids" in supported_params
def test_validate_environment_with_api_key(self):
headers = {}
api_key = "test-azure-key"
validated_headers = self.config.validate_environment(
headers=headers, api_key=api_key
)
assert "api-key" in validated_headers
assert validated_headers["api-key"] == api_key
def test_validate_environment_uses_azure_env_var(self, monkeypatch):
monkeypatch.setenv("AZURE_API_KEY", "env-azure-key")
headers = {}
validated_headers = self.config.validate_environment(headers=headers)
assert "api-key" in validated_headers
assert validated_headers["api-key"] == "env-azure-key"
def test_validate_environment_no_bearer_token(self):
"""Azure uses api-key header, not Authorization: Bearer."""
headers = {}
api_key = "azure-test-key"
validated_headers = self.config.validate_environment(
headers=headers, api_key=api_key
)
assert "Authorization" not in validated_headers
assert "api-key" in validated_headers
def test_get_complete_url_default_v1(self):
"""With default_api_version='v1', URL should include /openai/v1/containers."""
api_base = "https://my-resource.openai.azure.com"
litellm_params = {}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "/openai/v1/containers" in url
assert "my-resource.openai.azure.com" in url
def test_get_complete_url_with_explicit_api_version(self):
api_base = "https://my-resource.openai.azure.com"
litellm_params = {"api_version": "2025-01-01"}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "api-version=2025-01-01" in url
assert "/openai/containers" in url
def test_get_complete_url_with_latest_api_version(self):
api_base = "https://my-resource.openai.azure.com"
litellm_params = {"api_version": "latest"}
url = self.config.get_complete_url(
api_base=api_base, litellm_params=litellm_params
)
assert "/openai/v1/containers" in url
def test_get_complete_url_raises_without_api_base(self, monkeypatch):
monkeypatch.delenv("AZURE_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", None)
with pytest.raises(ValueError, match="api_base is required"):
self.config.get_complete_url(api_base=None, litellm_params={})
def test_transform_container_create_request(self):
from litellm.types.router import GenericLiteLLMParams
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
name = "My Azure Container"
optional_params = {
"expires_after": {"anchor": "last_active_at", "minutes": 30},
"file_ids": ["file_abc"],
}
data = self.config.transform_container_create_request(
name=name,
container_create_optional_request_params=optional_params,
litellm_params=litellm_params,
headers=headers,
)
assert data["name"] == name
assert data["expires_after"]["minutes"] == 30
assert data["file_ids"] == ["file_abc"]
def test_transform_container_create_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"id": "cntr_azure_123",
"object": "container",
"created_at": 1747857508,
"status": "running",
"expires_after": {"anchor": "last_active_at", "minutes": 30},
"last_active_at": 1747857508,
"name": "My Azure Container",
}
container = self.config.transform_container_create_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(container, ContainerObject)
assert container.id == "cntr_azure_123"
assert container.name == "My Azure Container"
assert container.status == "running"
def test_transform_container_list_request(self):
from litellm.types.router import GenericLiteLLMParams
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_list_request(
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
limit=5,
order="desc",
)
assert url == api_base
assert params["limit"] == "5"
assert params["order"] == "desc"
def test_transform_container_list_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"object": "list",
"data": [
{
"id": "cntr_1",
"object": "container",
"created_at": 1747857508,
"status": "running",
"expires_after": {"anchor": "last_active_at", "minutes": 20},
"last_active_at": 1747857508,
"name": "Container 1",
}
],
"first_id": "cntr_1",
"last_id": "cntr_1",
"has_more": False,
}
container_list = self.config.transform_container_list_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(container_list, ContainerListResponse)
assert len(container_list.data) == 1
assert container_list.first_id == "cntr_1"
def test_transform_container_retrieve_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_abc"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_retrieve_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert url == f"{api_base}/{container_id}"
assert params == {}
def test_transform_container_delete_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_del"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_delete_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert url == f"{api_base}/{container_id}"
assert params == {}
def test_transform_container_delete_response(self):
mock_response = MagicMock(spec=httpx.Response)
mock_response.json.return_value = {
"id": "cntr_azure_del",
"object": "container.deleted",
"deleted": True,
}
delete_result = self.config.transform_container_delete_response(
raw_response=mock_response, logging_obj=self.logging_obj
)
assert isinstance(delete_result, DeleteContainerResult)
assert delete_result.id == "cntr_azure_del"
assert delete_result.deleted is True
def test_transform_container_file_list_request(self):
from litellm.types.router import GenericLiteLLMParams
container_id = "cntr_azure_files"
api_base = "https://my-resource.openai.azure.com/openai/v1/containers"
litellm_params = GenericLiteLLMParams()
headers = {"api-key": "test-key"}
url, params = self.config.transform_container_file_list_request(
container_id=container_id,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
limit=10,
)
assert url == f"{api_base}/{container_id}/files"
assert params["limit"] == "10"
def test_transform_requests_preserve_query_string_after_path(self):
"""api-version must not appear before /{container_id}/... (Azure bases include ?)."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_r, _ = self.config.transform_container_retrieve_request(
container_id="cntr_x",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert (
url_r
== "https://my-resource.openai.azure.com/openai/v1/containers/cntr_x?api-version=v1"
)
url_fl, _ = self.config.transform_container_file_list_request(
container_id="cntr_x",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
assert (
url_fl
== "https://my-resource.openai.azure.com/openai/v1/containers/cntr_x/files?api-version=v1"
)
url_fc, _ = self.config.transform_container_file_content_request(
container_id="cntr_x",
file_id="cfile_y",
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
expected_fc = (
"https://my-resource.openai.azure.com/openai/v1/containers/"
"cntr_x/files/cfile_y/content?api-version=v1"
)
assert url_fc == expected_fc
assert url_fc.index("/content") < url_fc.index("?")
def test_provider_config_manager_returns_azure_config(self):
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
config = ProviderConfigManager.get_provider_container_config(
provider=LlmProviders.AZURE
)
assert config is not None
assert isinstance(config, AzureContainerConfig)
def test_proxy_handler_factory_returns_azure_config(self):
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
config = _get_container_provider_config("azure")
assert config is not None
assert isinstance(config, AzureContainerConfig)
def test_proxy_handler_factory_raises_for_unsupported_provider(self):
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
with pytest.raises(ValueError, match="Container API not supported"):
_get_container_provider_config("anthropic")
class TestAzureContainerKnownFailureRegressions:
"""Regression tests for real production / proxy failures (Azure containers).
1. **URL / api-version** — ``get_complete_url`` appends ``?api-version=…`` to the
container base. Naïve ``f\"{api_base}/…\"`` put the query *before* path segments,
e.g. ``…/containers?api-version=v1/cntr_…/files``, which Azure rejects
("API version not supported" / 404-style routing).
2. **Bare resource root** — ``AZURE_API_BASE`` is only the host (no ``?``). The
query appears only after LiteLLM builds the full container base; downstream
transforms must still append ``/cntr_…/files/…`` *before* the query string.
3. **File content path** — The worst case in logs was POST/GET logging showing
``…containers?api-version=v1/cntr_…/files/cfile_…/content``; correct wire shape is
``…containers/cntr_…/files/cfile_…/content?api-version=v1``.
"""
def setup_method(self):
self.config = AzureContainerConfig()
def test_regression_query_never_splits_before_container_segment(self):
"""Forbid the broken shape: …/containers?api-version=v1/cntr_…"""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
cid = "cntr_69d4f27de324819082c54f6aeaab6391056f5dbdf1fe2b02"
fid = "cfile_69d4f283bac0819094bfe7805a4f3ce8"
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_fc, _ = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
# Exact substring seen in broken logs
assert "containers?api-version=v1/" + cid not in url_fc
assert "containers?api-version=v1/cntr_" not in url_fc
parsed = urlparse(url_fc)
assert parsed.path == (
f"/openai/v1/containers/{cid}/files/{fid}/content"
)
assert parse_qs(parsed.query).get("api-version") == ["v1"]
assert url_fc.index("/content") < url_fc.index("?")
def test_regression_full_chain_bare_resource_root_like_env(self):
"""Mimics AZURE_API_BASE=https://resource.openai.azure.com — no ? in env."""
from litellm.types.router import GenericLiteLLMParams
resource_root = "https://my-resource.openai.azure.com"
container_base = self.config.get_complete_url(
api_base=resource_root,
litellm_params={},
)
assert "openai.azure.com" in container_base
assert "openai/v1/containers" in container_base or "/openai/containers" in container_base
cid = "cntr_livepath123"
fid = "cfile_live456"
url_fc, params = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=container_base,
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert cid in url_fc
assert fid in url_fc
parsed = urlparse(url_fc)
assert cid in parsed.path
assert "?" not in parsed.path
assert "/content" in parsed.path
assert url_fc.index(cid) < (url_fc.index("?") if "?" in url_fc else len(url_fc))
assert params == {}
def test_regression_all_crud_urls_with_azure_style_api_base(self):
"""Retrieve, delete, list files, and file content all keep ?api-version last."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://iamkankute-5584-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1"
)
cid = "cntr_69d4f1c5c6448190930a444af3f84f670b35dc2ee845cd1b"
fid = "cfile_69d4f1c97a1081908d22a9f56268c743"
litellm_params = GenericLiteLLMParams()
headers: dict = {}
url_r, _ = self.config.transform_container_retrieve_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_d, _ = self.config.transform_container_delete_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_lf, _ = self.config.transform_container_file_list_request(
container_id=cid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
url_fc, _ = self.config.transform_container_file_content_request(
container_id=cid,
file_id=fid,
api_base=api_base,
litellm_params=litellm_params,
headers=headers,
)
for name, u in (
("retrieve", url_r),
("delete", url_d),
("list_files", url_lf),
("file_content", url_fc),
):
assert f"containers?api-version=v1/{cid}" not in u, name
p = urlparse(u)
assert cid in p.path, name
assert "api-version" in p.query or "api-version=v1" in u, name
assert urlparse(url_fc).path.endswith(f"/{cid}/files/{fid}/content")
def test_regression_api_base_with_extra_query_params(self):
"""Multiple query params must stay at the end after path join."""
from litellm.types.router import GenericLiteLLMParams
api_base = (
"https://my-resource.openai.azure.com/openai/v1/containers"
"?api-version=v1&foo=bar"
)
cid = "cntr_x"
url_lf, _ = self.config.transform_container_file_list_request(
container_id=cid,
api_base=api_base,
litellm_params=GenericLiteLLMParams(),
headers={},
)
p = urlparse(url_lf)
assert p.path == f"/openai/v1/containers/{cid}/files"
qs = parse_qs(p.query)
assert qs.get("api-version") == ["v1"]
assert qs.get("foo") == ["bar"]
def test_regression_proxy_resolves_azure_text_same_as_azure(self):
"""Router/proxy treat azure_text like azure for container config."""
from litellm.proxy.container_endpoints.handler_factory import (
_get_container_provider_config,
)
c1 = _get_container_provider_config("azure")
c2 = _get_container_provider_config("azure_text")
assert type(c1) is type(c2)
assert isinstance(c1, AzureContainerConfig)
tests/test_litellm/containers/test_container_api.py
+71
View File
@@ -25,6 +25,7 @@ from litellm.containers.main import (
from litellm.main import base_llm_http_handler
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router import Router
from litellm.types.containers.main import (
ContainerListResponse,
@@ -220,6 +221,76 @@ class TestContainerAPI:
assert response.expires_after.minutes == 20
assert response.expires_after.anchor == "last_active_at"
def test_retrieve_container_reencodes_short_managed_id_for_routing(self):
"""Short cntr_ IDs must still re-encode output so follow-ups keep router affinity."""
short_managed_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id="router-gpt",
container_id="x",
)
assert short_managed_id.startswith("cntr_")
assert len(short_managed_id) < 100
mock_response = ContainerObject(
id="x",
object="container",
created_at=1747857508,
status="running",
expires_after={"anchor": "last_active_at", "minutes": 20},
last_active_at=1747857508,
name="Tiny",
)
with patch.object(
base_llm_http_handler,
"container_retrieve_handler",
return_value=mock_response,
) as mock_method:
response = retrieve_container(
container_id=short_managed_id,
custom_llm_provider="openai",
)
mock_method.assert_called_once()
assert mock_method.call_args.kwargs["container_id"] == "x"
assert response.id.startswith("cntr_")
decoded = ResponsesAPIRequestUtils._decode_container_id(response.id)
assert decoded.get("response_id") == "x"
assert decoded.get("model_id") == "router-gpt"
assert decoded.get("custom_llm_provider") == "azure"
def test_delete_container_reencodes_short_managed_id_for_routing(self):
"""Same as retrieve: short managed IDs must round-trip encoding on delete result."""
short_managed_id = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id="router-gpt",
container_id="z",
)
assert len(short_managed_id) < 100
mock_response = DeleteContainerResult(
id="z",
object="container.deleted",
deleted=True,
)
with patch.object(
base_llm_http_handler,
"container_delete_handler",
return_value=mock_response,
) as mock_method:
response = delete_container(
container_id=short_managed_id,
custom_llm_provider="openai",
)
mock_method.assert_called_once()
assert mock_method.call_args.kwargs["container_id"] == "z"
assert response.id.startswith("cntr_")
decoded = ResponsesAPIRequestUtils._decode_container_id(response.id)
assert decoded.get("response_id") == "z"
assert decoded.get("model_id") == "router-gpt"
@pytest.mark.asyncio
async def test_aretrieve_container_basic(self):
"""Test basic async container retrieval functionality."""
tests/test_litellm/containers/test_container_utils.py
+45 -2
View File
@@ -8,11 +8,17 @@ sys.path.insert(
) # Adds the parent directory to the system path
import litellm
from litellm.containers.utils import ContainerRequestUtils
from litellm.containers.utils import (
ContainerRequestUtils,
decode_managed_container_id_for_request,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.router import GenericLiteLLMParams
from litellm.llms.openai.containers.transformation import OpenAIContainerConfig
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
ContainerListOptionalRequestParams
ContainerListOptionalRequestParams,
DeleteContainerFileResponse,
)
@@ -228,3 +234,40 @@ class TestContainerRequestUtils:
)
assert result["expires_after"]["minutes"] == 15
def test_decode_managed_container_id_returns_provider_container_id(self):
"""Managed IDs must decode to the short ID sent on upstream requests."""
inner = "cntr_69d4ff00deadbeef"
managed = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="openai",
model_id=None,
container_id=inner,
)
assert len(managed) > len(inner)
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams()
original_id, provider, updated = decode_managed_container_id_for_request(
managed, "openai", litellm_params
)
assert original_id == inner
assert provider == "openai"
assert updated is litellm_params
class TestDeleteContainerFileResponseWireFormat:
"""OpenAI / Azure return ``container.file.deleted`` on DELETE file."""
def test_accepts_openai_dot_notation(self):
m = DeleteContainerFileResponse(
id="cfile_abc",
object="container.file.deleted",
deleted=True,
)
assert m.object == "container.file.deleted"
def test_accepts_legacy_underscore(self):
m = DeleteContainerFileResponse(
id="cfile_abc",
object="container_file.deleted",
deleted=True,
)
assert m.object == "container_file.deleted"
tests/test_litellm/responses/test_responses_utils.py
+29
View File
@@ -138,6 +138,35 @@ class TestResponsesAPIRequestUtils:
assert decoded.get("model_id") == "gpt-4o"
assert decoded.get("custom_llm_provider") == "openai"
def test_build_decode_container_id_omits_none_model_id(self):
"""model_id=None must not round-trip as the truthy string 'None'."""
encoded = ResponsesAPIRequestUtils._build_container_id(
custom_llm_provider="azure",
model_id=None,
container_id="cntr_upstream_abc",
)
assert "None" not in base64.b64decode(
encoded.replace("cntr_", "").encode("utf-8")
).decode("utf-8")
decoded = ResponsesAPIRequestUtils._decode_container_id(encoded)
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("model_id") is None
assert decoded.get("response_id") == "cntr_upstream_abc"
def test_decode_container_id_legacy_literal_none_model_id(self):
"""IDs encoded before the None fix should decode without a bogus model_id."""
legacy_inner = (
"litellm:custom_llm_provider:azure;model_id:None;container_id:cntr_x"
)
legacy_id = (
"cntr_"
+ base64.b64encode(legacy_inner.encode("utf-8")).decode("utf-8")
)
decoded = ResponsesAPIRequestUtils._decode_container_id(legacy_id)
assert decoded.get("model_id") is None
assert decoded.get("custom_llm_provider") == "azure"
assert decoded.get("response_id") == "cntr_x"
class TestResponseAPILoggingUtils:
def test_is_response_api_usage_true(self):