diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md new file mode 100644 index 0000000000..3697137686 --- /dev/null +++ b/docs/my-website/docs/providers/oci.md @@ -0,0 +1,84 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Oracle Cloud Infrastructure (OCI) +LiteLLM supports the following models for OCI on-demand GenAI API. + +Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region. + +- `cohere.command-a-03-2025` +- `cohere.command-r-08-2024` +- `cohere.command-plus-latest` (alias `cohere.command-r-plus-08-2024`) +- `cohere.command-r-16k` (deprecated) +- `cohere.command-r-plus` (deprecated) + +- `meta.llama-4-maverick-17b-128e-instruct-fp8` +- `meta.llama-4-scout-17b-16e-instruct` +- `meta.llama-3.3-70b-instruct` +- `meta.llama-3.2-90b-vision-instruct` +- `meta.llama-3.2-11b-vision-instruct` +- `meta.llama-3.1-405b-instruct` +- `meta.llama-3.1-70b-instruct` +- `meta.llama-3-70b-instruct` + +- `xai.grok-4` +- `xai.grok-3` +- `xai.grok-3-fast` +- `xai.grok-3-mini` +- `xai.grok-3-mini-fast` + +## Authentication + +LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters: + +- `user` +- `fingerprint` +- `tenancy` +- `region` +- `key_file` + +## Usage + +Input the parameters obtained from the OCI signing key creation process into the `completion` function. + +```python +import os +from litellm import completion + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_region=, + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +print(response) +``` + + +## Usage - Streaming +Just set `stream=True` when calling completion. + +```python +import os +from litellm import completion + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + stream=True, + oci_region=, + oci_user=, + oci_fingerprint=, + oci_tenancy=, + oci_key=, + oci_compartment_id=, +) +for chunk in response: + print(chunk["choices"][0]["delta"]["content"]) # same as openai format +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 83ebfe29dc..bffa8a91b6 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -469,7 +469,8 @@ const sidebars = { "providers/featherless_ai", "providers/nebius", "providers/dashscope", - "providers/bytez" + "providers/bytez", + "providers/oci", ], }, { diff --git a/litellm/__init__.py b/litellm/__init__.py index 8cf7d3922a..03cec9afb1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1199,6 +1199,7 @@ from .llms.nebius.chat.transformation import NebiusConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig from .llms.v0.chat.transformation import V0ChatConfig +from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig diff --git a/litellm/constants.py b/litellm/constants.py index b25e2ff0fb..27cea0eb04 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -279,6 +279,7 @@ LITELLM_CHAT_PROVIDERS = [ "dashscope", "moonshot", "v0", + "oci", "morph", "lambda_ai", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 4e0a2efb0c..702196a7f0 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -356,6 +356,8 @@ def get_llm_provider( # noqa: PLR0915 # bytez models elif model.startswith("bytez/"): custom_llm_provider = "bytez" + elif model.startswith("oci/"): + custom_llm_provider = "oci" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py new file mode 100644 index 0000000000..e08c1d1240 --- /dev/null +++ b/litellm/llms/oci/chat/transformation.py @@ -0,0 +1,850 @@ +import base64 +import datetime +import hashlib +from urllib.parse import urlparse +import litellm +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa + +import httpx + +from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, + version, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import LlmProviders +from litellm.utils import ( + ChatCompletionMessageToolCall, + CustomStreamWrapper, + ModelResponse, + Usage, +) +from litellm.types.llms.oci import ( + OCIChatRequestPayload, + OCICompletionPayload, + OCICompletionResponse, + OCIContentPartUnion, + OCIImageContentPart, + OCIMessage, + OCIRoles, + OCIServingMode, + OCIStreamChunk, + OCITextContentPart, + OCIToolCall, + OCIToolDefinition, + OCIVendors, +) +from litellm.llms.oci.common_utils import OCIError +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StreamingChoices, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +def sha256_base64(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return base64.b64encode(digest).decode() + + +def build_signature_string(method, path, headers, signed_headers): + lines = [] + for header in signed_headers: + if header == "(request-target)": + value = f"{method.lower()} {path}" + else: + value = headers[header] + lines.append(f"{header}: {value}") + return "\n".join(lines) + + +def load_private_key_from_str(key_str: str): + key = serialization.load_pem_private_key( + key_str.encode("utf-8"), + password=None, + ) + if not isinstance(key, rsa.RSAPrivateKey): + raise TypeError( + "The provided private key is not an RSA key, which is required for OCI signing." + ) + return key + + +def get_vendor_from_model(model: str) -> OCIVendors: + """ + Extracts the vendor from the model name. + Args: + model (str): The model name. + Returns: + str: The vendor name. + """ + vendor = model.split(".")[0].lower() + if vendor == "cohere": + return OCIVendors.COHERE + else: + return OCIVendors.GENERIC + + +# 5 minute timeout (models may need to load) +STREAMING_TIMEOUT = 60 * 5 + + +class OCIChatConfig(BaseConfig): + """ + Configuration class for OCI's API interface. + """ + + def __init__( + self, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + # mark the class as using a custom stream wrapper because the default only iterates on lines + setattr(self.__class__, "has_custom_stream_wrapper", True) + + self.openai_to_oci_generic_param_map = { + "stream": "isStream", + "max_tokens": "maxTokens", + "max_completion_tokens": "maxTokens", + "temperature": "temperature", + "tools": "tools", + "frequency_penalty": "frequencyPenalty", + "logprobs": "logProbs", + "logit_bias": "logitBias", + "n": "numGenerations", + "presence_penalty": "presencePenalty", + "seed": "seed", + "stop": "stop", + "tool_choice": "toolChoice", + "top_p": "topP", + "max_retries": False, + "top_logprobs": False, + "modalities": False, + "prediction": False, + "stream_options": False, + "function_call": False, + "functions": False, + "extra_headers": False, + "parallel_tool_calls": False, + "audio": False, + "web_search_options": False, + } + + def get_supported_openai_params(self, model: str) -> List[str]: + supported_params = [] + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + for key, value in open_ai_to_oci_param_map.items(): + if value: + supported_params.append(key) + + return supported_params + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + adapted_params = {} + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + + all_params = {**non_default_params, **optional_params} + + for key, value in all_params.items(): + alias = open_ai_to_oci_param_map.get(key) + + if alias is False: + if drop_params: + continue + + raise Exception(f"param `{key}` is not supported on OCI") + + if alias is None: + adapted_params[key] = value + continue + + adapted_params[alias] = value + + return adapted_params + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Some providers like Bedrock require signing the request. The sign request funtion needs access to `request_data` and `complete_url` + Args: + headers: dict + optional_params: dict + request_data: dict - the request body being sent in http request + api_base: str - the complete url being sent in http request + Returns: + dict - the signed headers + """ + import json + + oci_region = optional_params.get("oci_region", "us-ashburn-1") + api_base = ( + api_base + or litellm.api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + + if not oci_user or not oci_fingerprint or not oci_tenancy or not oci_key: + raise Exception( + "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key" + ) + + method = str(optional_params.get("method", "POST")).upper() + body = json.dumps(request_data).encode("utf-8") + parsed = urlparse(api_base) + path = parsed.path or "/" + host = parsed.netloc + + date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") + content_type = headers.get("content-type", "application/json") + content_length = str(len(body)) + x_content_sha256 = sha256_base64(body) + + headers_to_sign = { + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + + signed_headers = [ + "date", + "(request-target)", + "host", + "content-length", + "content-type", + "x-content-sha256", + ] + signing_string = build_signature_string( + method, path, headers_to_sign, signed_headers + ) + + private_key = load_private_key_from_str(oci_key) + signature = private_key.sign( + signing_string.encode("utf-8"), + padding.PKCS1v15(), + hashes.SHA256(), + ) + signature_b64 = base64.b64encode(signature).decode() + + key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" + + authorization = ( + 'Signature version="1",' + f'keyId="{key_id}",' + 'algorithm="rsa-sha256",' + f'headers="{" ".join(signed_headers)}",' + f'signature="{signature_b64}"' + ) + + headers.update( + { + "authorization": authorization, + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + ) + + return headers, None + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + oci_region = optional_params.get("oci_region", "us-ashburn-1") + api_base = ( + api_base + or litellm.api_base + or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" + ) + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + oci_compartment_id = optional_params.get("oci_compartment_id") + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not oci_key + or not oci_compartment_id + ): + raise Exception( + "Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" + ) + + if not api_base: + raise Exception( + "Either `api_base` must be provided or `litellm.api_base` must be set. Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." + ) + + headers.update( + { + "content-type": "application/json", + "user-agent": f"litellm/{version}", + } + ) + + if not messages: + raise Exception( + "kwarg `messages` must be an array of messages that follow the openai chat standard" + ) + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + oci_region = optional_params.get("oci_region", "us-ashburn-1") + return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat" + + def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict: + selected_params = {} + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + + for value in open_ai_to_oci_param_map.values(): + if value in optional_params: + selected_params[value] = optional_params[value] + if "tools" in selected_params: + selected_params["tools"] = adapt_tool_definition_to_oci_standard( + selected_params["tools"], vendor + ) + return selected_params + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + oci_compartment_id = optional_params.get("oci_compartment_id", None) + if not oci_compartment_id: + raise Exception("kwarg `oci_compartment_id` is required for OCI requests") + + vendor = get_vendor_from_model(model) + + if vendor == OCIVendors.COHERE: + raise Exception( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + data = OCICompletionPayload( + compartmentId=oci_compartment_id, + servingMode=OCIServingMode( + servingType="ON_DEMAND", + modelId=model, + ), + chatRequest=OCIChatRequestPayload( + apiFormat=vendor.value, + messages=adapt_messages_to_generic_oci_standard(messages), + **self._get_optional_params(vendor, optional_params), + ), + ) + + return data.model_dump(exclude_none=True) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + json = raw_response.json() # noqa: F811 + + error = json.get("error") + + if error is not None: + raise OCIError( + message=str(json["error"]), + status_code=raw_response.status_code, + ) + + if not isinstance(json, dict): + raise OCIError( + message="Invalid response format from OCI", + status_code=raw_response.status_code, + ) + + try: + completion_response = OCICompletionResponse(**json) + except TypeError as e: + raise OCIError( + message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", + status_code=raw_response.status_code, + ) + + vendor = get_vendor_from_model(model) + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + iso_str = completion_response.chatResponse.timeCreated + dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) + model_response.created = int(dt.timestamp()) + + model_response.model = completion_response.modelId + + message = model_response.choices[0].message # type: ignore + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + response_message = completion_response.chatResponse.choices[0].message + if response_message.content and response_message.content[0].type == "TEXT": + message.content = response_message.content[0].text + if response_message.toolCalls: + message.tool_calls = adapt_tools_to_openai_standard( + response_message.toolCalls + ) + + usage = Usage( + prompt_tokens=completion_response.chatResponse.usage.promptTokens, + completion_tokens=completion_response.chatResponse.usage.completionTokens, + total_tokens=completion_response.chatResponse.usage.totalTokens, + ) + model_response.usage = usage # type: ignore + + model_response._hidden_params["additional_headers"] = raw_response.headers + + return model_response + + @track_llm_api_timing() + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "OCIStreamWrapper": + if "stream" in data: + del data["stream"] + if client is None or isinstance(client, AsyncHTTPHandler): + client = _get_httpx_client(params={}) + + try: + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + timeout=STREAMING_TIMEOUT, + ) + except httpx.HTTPStatusError as e: + raise OCIError(status_code=e.response.status_code, message=e.response.text) + + if response.status_code != 200: + raise OCIError(status_code=response.status_code, message=response.text) + + completion_stream = response.iter_text() + + streaming_response = OCIStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return streaming_response + + @track_llm_api_timing() + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "OCIStreamWrapper": + if "stream" in data: + del data["stream"] + + if client is None or isinstance(client, HTTPHandler): + client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={}) + + try: + response = await client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + timeout=STREAMING_TIMEOUT, + ) + except httpx.HTTPStatusError as e: + raise OCIError(status_code=e.response.status_code, message=e.response.text) + + if response.status_code != 200: + raise OCIError(status_code=response.status_code, message=response.text) + + completion_stream = response.aiter_text() + + streaming_response = OCIStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return streaming_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return OCIError(status_code=status_code, message=error_message) + + +open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { + "system": "SYSTEM", + "user": "USER", + "assistant": "ASSISTANT", + "tool": "TOOL", +} + + +def adapt_messages_to_generic_oci_standard_content_message( + role: str, content: str | list +) -> OCIMessage: + new_content: list[OCIContentPartUnion] = [] + if isinstance(content, str): + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=None, + ) + + # content is a list of content items: + # [ + # {"type": "text", "text": "Hello"}, + # {"type": "image_url", "image_url": "https://example.com/image.png"} + # ] + for content_item in content: + if not isinstance(content_item, dict): + raise Exception("Each content item must be a dictionary") + + type = content_item.get("type") + if not isinstance(type, str): + raise Exception("Prop `type` is not a string") + + if type not in ["text", "image_url"]: + raise Exception(f"Prop `{type}` is not supported") + + if type == "text": + text = content_item.get("text") + if not isinstance(text, str): + raise Exception("Prop `text` is not a string") + new_content.append(OCITextContentPart(text=text)) + + elif type == "image_url": + image_url = content_item.get("image_url") + if not isinstance(image_url, str): + raise Exception("Prop `image_url` is not a string") + new_content.append(OCIImageContentPart(imageUrl=image_url)) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=new_content, + toolCalls=None, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_call( + role: str, tool_calls: list +) -> OCIMessage: + tool_calls_formated = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + raise Exception("Each tool call must be a dictionary") + + if tool_call.get("type") != "function": + raise Exception("OCI only supports function tools") + + tool_call_id = tool_call.get("id") + if not isinstance(tool_call_id, str): + raise Exception("Prop `id` is not a string") + + tool_function = tool_call.get("function") + if not isinstance(tool_function, dict): + raise Exception("Prop `function` is not a dictionary") + + function_name = tool_function.get("name") + if not isinstance(function_name, str): + raise Exception("Prop `name` is not a string") + + arguments = tool_call["function"].get("arguments", "{}") + if not isinstance(arguments, str): + raise Exception("Prop `arguments` is not a string") + + # tool_calls_formated.append(OCIToolCall( + # id=tool_call_id, + # type="FUNCTION", + # function=OCIFunction( + # name=function_name, + # arguments=arguments + # ) + # )) + + tool_calls_formated.append( + OCIToolCall( + id=tool_call_id, + type="FUNCTION", + name=function_name, + arguments=arguments, + ) + ) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=None, + toolCalls=tool_calls_formated, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_response( + role: str, tool_call_id: str, content: str +) -> OCIMessage: + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=tool_call_id, + ) + + +def adapt_messages_to_generic_oci_standard( + messages: List[AllMessageValues], +) -> List[OCIMessage]: + new_messages = [] + for message in messages: + role = message["role"] + content = message.get("content") + tool_calls = message.get("tool_calls") + tool_call_id = message.get("tool_call_id") + + if role in ["system", "user", "assistant"] and content is not None: + if not isinstance(content, (str, list)): + raise Exception( + "Prop `content` must be a string or a list of content items" + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_content_message(role, content) + ) + + elif role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise Exception("Prop `tool_calls` must be a list of tool calls") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role == "tool": + if not isinstance(tool_call_id, str): + raise Exception("Prop `tool_call_id` is required and must be a string") + if not isinstance(content, str): + raise Exception("Prop `content` is not a string") + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_response( + role, tool_call_id, content + ) + ) + + return new_messages + + +def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors): + new_tools = [] + if vendor == OCIVendors.COHERE: + raise ValueError( + "Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly." + ) + else: + for tool in tools: + if tool["type"] != "function": + raise Exception("OCI only supports function tools") + + tool_function = tool.get("function") + if not isinstance(tool_function, dict): + raise Exception("Prop `function` is not a dictionary") + + new_tool = OCIToolDefinition( + type="FUNCTION", + name=tool_function.get("name"), + description=tool_function.get("description", ""), + parameters=tool_function.get("parameters", {}), + ) + new_tools.append(new_tool) + + return new_tools + + +def adapt_tools_to_openai_standard( + tools: list[OCIToolCall], +) -> list[ChatCompletionMessageToolCall]: + new_tools = [] + for tool in tools: + new_tool = ChatCompletionMessageToolCall( + id=tool.id, + type="function", + function={ + "name": tool.name, + "arguments": tool.arguments, + }, + ) + new_tools.append(new_tool) + return new_tools + + +class OCIStreamWrapper(CustomStreamWrapper): + """ + Custom stream wrapper for OCI responses. + This class is used to handle streaming responses from OCI's API. + """ + + def __init__( + self, + **kwargs: Any, + ): + super().__init__(**kwargs) + + def chunk_creator(self, chunk: Any): + if not isinstance(chunk, str): + raise ValueError(f"Chunk is not a string: {chunk}") + if not chunk.startswith("data:"): + raise ValueError(f"Chunk does not start with 'data:': {chunk}") + dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON + try: + typed_chunk = OCIStreamChunk(**dict_chunk) + except TypeError as e: + raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}") + + if typed_chunk.index is None: + typed_chunk.index = 0 + + text = "" + if typed_chunk.message and typed_chunk.message.content: + for item in typed_chunk.message.content: + if isinstance(item, OCITextContentPart): + text += item.text + elif isinstance(item, OCIImageContentPart): + raise ValueError( + "OCI does not support image content in streaming responses" + ) + else: + raise ValueError( + f"Unsupported content type in OCI response: {item.type}" + ) + + tool_calls = None + if typed_chunk.message and typed_chunk.message.toolCalls: + tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index if typed_chunk.index else 0, + delta=Delta( + content=text, + tool_calls=[tool.model_dump() for tool in tool_calls] + if tool_calls + else None, + provider_specific_fields=None, # OCI does not have provider specific fields in the response + thinking_blocks=None, # OCI does not have thinking blocks in the response + reasoning_content=None, # OCI does not have reasoning content in the response + ), + finish_reason=typed_chunk.finishReason, + ) + ] + ) diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py new file mode 100644 index 0000000000..661a6c89e4 --- /dev/null +++ b/litellm/llms/oci/common_utils.py @@ -0,0 +1,19 @@ +from typing import Optional + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class OCIError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: Optional[httpx.Headers] = None, + ): + super().__init__( + status_code=status_code, + message=message, + headers=headers, + ) diff --git a/litellm/main.py b/litellm/main.py index 4fc81cce6d..a6583ab059 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -151,6 +151,7 @@ from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion +from .llms.oci.chat.transformation import OCIChatConfig from .llms.ollama.completion import handler as ollama from .llms.oobabooga.chat import oobabooga from .llms.openai.completion.handler import OpenAITextCompletion @@ -252,6 +253,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler() sagemaker_chat_completion = SagemakerChatHandler() bytez_transformation = BytezChatConfig() +oci_transformation = OCIChatConfig() ####### COMPLETION ENDPOINTS ################ @@ -2399,6 +2401,24 @@ def completion( # type: ignore # noqa: PLR0915 encoding=encoding, stream=stream, ) + elif custom_llm_provider == "oci": + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + ) elif custom_llm_provider == "oobabooga": custom_llm_provider = "oobabooga" model_response = oobabooga.completion( diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py new file mode 100644 index 0000000000..cb76bf8747 --- /dev/null +++ b/litellm/types/llms/oci.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from typing import Any, Literal, Union + +from pydantic import BaseModel +from enum import Enum + +OCIRoles = Literal["SYSTEM", "USER", "ASSISTANT", "TOOL"] + + +class OCIVendors(Enum): + """ + A class to hold the vendor names for OCI models. + This is used to map model names to their respective vendors. + """ + + COHERE = "COHERE" + GENERIC = "GENERIC" + + +# --- Base Models and Content Parts --- + + +class OCIContentPart(BaseModel): + """Base model for content parts in an OCI message.""" + + type: str + + +class OCITextContentPart(OCIContentPart): + """Text content part for the OCI API.""" + + type: Literal["TEXT"] = "TEXT" + text: str + + +class OCIImageContentPart(OCIContentPart): + """Image content part for the OCI API.""" + + type: Literal["IMAGE"] = "IMAGE" + imageUrl: str + + +OCIContentPartUnion = Union[OCITextContentPart, OCIImageContentPart] + +# --- Models for Tools and Tool Calls --- + + +class OCIToolCall(BaseModel): + """Represents a tool call made by the model.""" + + id: str + type: Literal["FUNCTION"] = "FUNCTION" + name: str + arguments: str # Arguments should be a JSON-serialized string + + +class OCIToolDefinition(BaseModel): + """Defines a tool that can be used by the model.""" + + type: Literal["FUNCTION"] = "FUNCTION" + name: str | None = None + description: str | None = None + parameters: dict | None = None + + +# --- Message Models (Request and Response) --- + + +class OCIMessage(BaseModel): + """Model for a single message in the request/response payload.""" + + role: OCIRoles + content: list[OCIContentPartUnion] | None = None + toolCalls: list[OCIToolCall] | None = None + toolCallId: str | None = None + + +# --- Request Payload Models --- + + +class OCIChatRequestPayload(BaseModel): + """Internal 'chatRequest' payload for the OCI API.""" + + apiFormat: str + messages: list[OCIMessage] + tools: list[OCIToolDefinition] | None = None + isStream: bool = False + numGenerations: int | None = None + maxTokens: int | None = None + temperature: float | None = None + topP: float | None = None + stop: list[str] | None = None + seed: int | None = None + frequencyPenalty: float | None = None + presencePenalty: float | None = None + + +class OCIServingMode(BaseModel): + """Defines the serving mode and the model to be used.""" + + servingType: str + modelId: str + + +class OCICompletionPayload(BaseModel): + """Pydantic model for the complete OCI chat request body.""" + + compartmentId: str + servingMode: OCIServingMode + chatRequest: OCIChatRequestPayload + + +# --- API Response Models (Non-streaming) --- + + +class OCICompletionTokenDetails(BaseModel): + """Completion token details in the OCI response.""" + + acceptedPredictionTokens: int + reasoningTokens: int + + +class OCIPropmtTokensDetails(BaseModel): + """Prompt token details in the OCI response.""" + + cachedTokens: int + + +class OCIResponseUsage(BaseModel): + """Token usage in the OCI response.""" + + promptTokens: int + completionTokens: int + totalTokens: int + completionTokensDetails: OCICompletionTokenDetails + promptTokensDetails: OCIPropmtTokensDetails + + +class OCIResponseChoice(BaseModel): + """A completion choice in the OCI response.""" + + index: int + message: OCIMessage + finishReason: str | None + logprobs: dict[str, Any] | None = None + + +class OCIChatResponse(BaseModel): + """The 'chatResponse' object in the OCI response.""" + + apiFormat: str + timeCreated: str + choices: list[OCIResponseChoice] + usage: OCIResponseUsage + + +class OCICompletionResponse(BaseModel): + """Model for the complete non-streaming OCI response body.""" + + modelId: str + modelVersion: str + chatResponse: OCIChatResponse + + +# --- API Response Models (Streaming) --- + + +class OCIStreamDelta(BaseModel): + """The content delta in a streaming chunk.""" + + content: list[OCIContentPartUnion] | None = None + role: str | None = None + toolCalls: list[OCIToolCall] | None = None + + +class OCIStreamChunk(BaseModel): + """Model for a single SSE event chunk from OCI.""" + + finishReason: str | None = None + message: OCIStreamDelta | None = None + pad: str | None = None + index: int | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c5af6e4ed0..f8df5781de 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2321,6 +2321,7 @@ class LlmProviders(str, Enum): PG_VECTOR = "pg_vector" HYPERBOLIC = "hyperbolic" RECRAFT = "recraft" + OCI = "oci" AUTO_ROUTER = "auto_router" DOTPROMPT = "dotprompt" diff --git a/litellm/utils.py b/litellm/utils.py index 9e9a112f30..a9695e14c0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6916,6 +6916,8 @@ class ProviderConfigManager: return litellm.OpenAIGPTConfig() elif litellm.LlmProviders.NSCALE == provider: return litellm.NscaleConfig() + elif litellm.LlmProviders.OCI == provider: + return litellm.OCIChatConfig() elif litellm.LlmProviders.HYPERBOLIC == provider: return litellm.HyperbolicChatConfig() return None diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py new file mode 100644 index 0000000000..d536925350 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -0,0 +1,297 @@ +import datetime +import os +import sys +import httpx +import pytest +import json + +import litellm + +# Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm import ModelResponse +from litellm.llms.oci.chat.transformation import OCIChatConfig, version + +TEST_MODEL_NAME = "xai.grok-4" +TEST_MODEL = f"oci/{TEST_MODEL_NAME}" +TEST_MESSAGES = [{"role": "user", "content": "Hello, how are you?"}] +TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx" +TEST_OCI_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx", + "oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90", + "oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx", + "oci_compartment_id": TEST_COMPARTMENT_ID, + "oci_key": "" +} + + +class TestOCIChatConfig: + def test_validate_environment_with_oci_region(self): + config = OCIChatConfig() + + headers = {} + + result = config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=TEST_MESSAGES, # type: ignore + optional_params=TEST_OCI_PARAMS, + litellm_params={}, + ) + + assert result["content-type"] == "application/json" + assert result["user-agent"] == f"litellm/{version}" + + def test_missing_oci_auth_parameters(self): + optional_params = TEST_OCI_PARAMS.copy() + optional_params.pop("oci_region") + + # Remove optional_params one by one and verify that an exception is raised + for key in optional_params.keys(): + modified_params = optional_params.copy() + del modified_params[key] + + with pytest.raises(Exception) as excinfo: + config = OCIChatConfig() + + headers = {} + + config.validate_environment( + headers=headers, + model=TEST_MODEL, + messages=TEST_MESSAGES, # type: ignore + optional_params=modified_params, + api_base="https://api.oci.example.com", + litellm_params={}, + ) + + assert f"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" in str(excinfo.value) + + def test_transform_request_simple(self): + """ + Tests if a simple request is transformed correctly. + """ + config = OCIChatConfig() + optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + expected_output = { + "compartmentId": TEST_COMPARTMENT_ID, + "servingMode": {"servingType": "ON_DEMAND", "modelId": TEST_MODEL_NAME}, + "chatRequest": { + "apiFormat": "GENERIC", + "isStream": False, + "messages": [ + { + "role": "USER", + "content": [{"type": "TEXT", "text": "Hello, how are you?"}], + } + ], + }, + } + assert transformed_request == expected_output + + def test_transform_request_with_tools(self): + """ + Tests if a request with tools is transformed correctly. + """ + config = OCIChatConfig() + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + "required": ["location"], + }, + }, + } + ] + optional_params = { + "oci_compartment_id": TEST_COMPARTMENT_ID, + "tools": tools, + } + transformed_request = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + assert "tools" in transformed_request["chatRequest"] + assert transformed_request["chatRequest"]["tools"][0]["name"] == "get_current_weather" + assert transformed_request["chatRequest"]["tools"][0]["type"] == "FUNCTION" + assert transformed_request["chatRequest"]["tools"][0]["description"] == "Get the current weather in a given location" + assert transformed_request["chatRequest"]["tools"][0]["parameters"] is not None + + def test_transform_response_simple_text(self): + """ + Tests if a simple text response is transformed correctly. + """ + config = OCIChatConfig() + created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z") + mock_oci_response = { + "modelId": TEST_MODEL_NAME, + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "I am doing well, thank you!"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 10, + "completionTokens": 20, + "totalTokens": 30, + "completionTokensDetails": { + "acceptedPredictionTokens": 20, + "reasoningTokens": 20, + }, + "promptTokensDetails": { + "cachedTokens": 10, + }, + }, + }, + } + response = httpx.Response( + status_code=200, json=mock_oci_response, headers={"Content-Type": "application/json"} + ) + result = config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert len(result.choices) == 1 + assert isinstance(result.choices[0], litellm.Choices) + assert result.choices[0].message + assert result.choices[0].message.content == "I am doing well, thank you!" + assert result.choices[0].finish_reason == "stop" + assert result.model == TEST_MODEL_NAME + assert hasattr(result, "usage") + assert isinstance(result.usage, litellm.Usage) # type: ignore + assert result.usage.prompt_tokens == 10 # type: ignore + assert result.usage.completion_tokens == 20 # type: ignore + assert result.usage.total_tokens == 30 # type: ignore + + def test_transform_response_with_tool_calls(self): + """ + Tests if a response with tool calls is transformed correctly. + """ + config = OCIChatConfig() + created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z") + mock_oci_response = { + "modelId": TEST_MODEL_NAME, + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": None, + "toolCalls": [ + { + "id": "call_abc123", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Vila Velha, BR"}', + } + ], + }, + "finishReason": "stop", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 10, + "completionTokens": 20, + "totalTokens": 30, + "completionTokensDetails": { + "acceptedPredictionTokens": 20, + "reasoningTokens": 20, + }, + "promptTokensDetails": { + "cachedTokens": 10, + }, + }, + }, + } + response = httpx.Response(status_code=200, json=mock_oci_response) + model_response = ModelResponse( + choices=[litellm.Choices(index=0, message=litellm.Message())] + ) + + result = config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=model_response, + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + # General assertions + assert isinstance(result, ModelResponse) + assert len(result.choices) == 1 + + choice = result.choices[0] + assert isinstance(choice, litellm.Choices) + assert choice.finish_reason == "stop" + + # Message and tool_calls assertions + message = choice.message + assert isinstance(message, litellm.Message) + assert hasattr(message, "tool_calls") + assert isinstance(message.tool_calls, list) + assert len(message.tool_calls) == 1 + + # Specific tool_call assertions + tool_call = message.tool_calls[0] + assert isinstance(tool_call, litellm.utils.ChatCompletionMessageToolCall) + assert tool_call.id == "call_abc123" + assert tool_call.type == "function" + assert tool_call.function["name"] == "get_weather" + assert tool_call.function["arguments"] == '{"location": "Vila Velha, BR"}' + + # Usage assertions + assert hasattr(result, "usage") + usage = result.usage # type: ignore + assert isinstance(usage, litellm.Usage) # type: ignore + assert usage.prompt_tokens == 10 # type: ignore + assert usage.completion_tokens == 20 # type: ignore + assert usage.total_tokens == 30 # type: ignore