mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-05 06:22:12 +00:00
Support OCI provider (#13206)
* create OCI required files * request and response conversion for non-streaming chat * support tool calling with OCI generic API without streaming * adaptation of api call for generic and cohere format * include tool calls and responses in generic api and dropping support for cohere * fix invalid content-length error * support streaming for generic api * fix auth error when using acompletion with streaming * refactor: use base_llm_http_handler and include API type definitions * update types and add type safety in different methods * fix OCIFunction format * create custom stream wrapper for decoding OCI stream * remove unused files * create unit tests for OCI * lint the code * remove manual test * docs: update the docs to include OCI
This commit is contained in:
breno-aumo
@@ -0,0 +1,84 @@
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Oracle Cloud Infrastructure (OCI)
|
||||
LiteLLM supports the following models for OCI on-demand GenAI API.
|
||||
|
||||
Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm) to see if the model is available for your region.
|
||||
|
||||
- `cohere.command-a-03-2025`
|
||||
- `cohere.command-r-08-2024`
|
||||
- `cohere.command-plus-latest` (alias `cohere.command-r-plus-08-2024`)
|
||||
- `cohere.command-r-16k` (deprecated)
|
||||
- `cohere.command-r-plus` (deprecated)
|
||||
|
||||
- `meta.llama-4-maverick-17b-128e-instruct-fp8`
|
||||
- `meta.llama-4-scout-17b-16e-instruct`
|
||||
- `meta.llama-3.3-70b-instruct`
|
||||
- `meta.llama-3.2-90b-vision-instruct`
|
||||
- `meta.llama-3.2-11b-vision-instruct`
|
||||
- `meta.llama-3.1-405b-instruct`
|
||||
- `meta.llama-3.1-70b-instruct`
|
||||
- `meta.llama-3-70b-instruct`
|
||||
|
||||
- `xai.grok-4`
|
||||
- `xai.grok-3`
|
||||
- `xai.grok-3-fast`
|
||||
- `xai.grok-3-mini`
|
||||
- `xai.grok-3-mini-fast`
|
||||
|
||||
## Authentication
|
||||
|
||||
LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters:
|
||||
|
||||
- `user`
|
||||
- `fingerprint`
|
||||
- `tenancy`
|
||||
- `region`
|
||||
- `key_file`
|
||||
|
||||
## Usage
|
||||
|
||||
Input the parameters obtained from the OCI signing key creation process into the `completion` function.
|
||||
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
messages = [{"role": "user", "content": "Hey! how's it going?"}]
|
||||
response = completion(
|
||||
model="oci/xai.grok-4",
|
||||
messages=messages,
|
||||
oci_region=<your_oci_region>,
|
||||
oci_user=<your_oci_user>,
|
||||
oci_fingerprint=<your_oci_fingerprint>,
|
||||
oci_tenancy=<your_oci_tenancy>,
|
||||
oci_key=<string_with_content_of_oci_key>,
|
||||
oci_compartment_id=<oci_compartment_id>,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
|
||||
## Usage - Streaming
|
||||
Just set `stream=True` when calling completion.
|
||||
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
|
||||
messages = [{"role": "user", "content": "Hey! how's it going?"}]
|
||||
response = completion(
|
||||
model="oci/xai.grok-4",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
oci_region=<your_oci_region>,
|
||||
oci_user=<your_oci_user>,
|
||||
oci_fingerprint=<your_oci_fingerprint>,
|
||||
oci_tenancy=<your_oci_tenancy>,
|
||||
oci_key=<string_with_content_of_oci_key>,
|
||||
oci_compartment_id=<oci_compartment_id>,
|
||||
)
|
||||
for chunk in response:
|
||||
print(chunk["choices"][0]["delta"]["content"]) # same as openai format
|
||||
```
|
||||
@@ -469,7 +469,8 @@ const sidebars = {
|
||||
"providers/featherless_ai",
|
||||
"providers/nebius",
|
||||
"providers/dashscope",
|
||||
"providers/bytez"
|
||||
"providers/bytez",
|
||||
"providers/oci",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1199,6 +1199,7 @@ from .llms.nebius.chat.transformation import NebiusConfig
|
||||
from .llms.dashscope.chat.transformation import DashScopeChatConfig
|
||||
from .llms.moonshot.chat.transformation import MoonshotChatConfig
|
||||
from .llms.v0.chat.transformation import V0ChatConfig
|
||||
from .llms.oci.chat.transformation import OCIChatConfig
|
||||
from .llms.morph.chat.transformation import MorphChatConfig
|
||||
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
|
||||
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
|
||||
@@ -279,6 +279,7 @@ LITELLM_CHAT_PROVIDERS = [
|
||||
"dashscope",
|
||||
"moonshot",
|
||||
"v0",
|
||||
"oci",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
]
|
||||
|
||||
@@ -356,6 +356,8 @@ def get_llm_provider( # noqa: PLR0915
|
||||
# bytez models
|
||||
elif model.startswith("bytez/"):
|
||||
custom_llm_provider = "bytez"
|
||||
elif model.startswith("oci/"):
|
||||
custom_llm_provider = "oci"
|
||||
if not custom_llm_provider:
|
||||
if litellm.suppress_debug_info is False:
|
||||
print() # noqa
|
||||
|
||||
@@ -0,0 +1,850 @@
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
import litellm
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding, rsa
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
version,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
CustomStreamWrapper,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
from litellm.types.llms.oci import (
|
||||
OCIChatRequestPayload,
|
||||
OCICompletionPayload,
|
||||
OCICompletionResponse,
|
||||
OCIContentPartUnion,
|
||||
OCIImageContentPart,
|
||||
OCIMessage,
|
||||
OCIRoles,
|
||||
OCIServingMode,
|
||||
OCIStreamChunk,
|
||||
OCITextContentPart,
|
||||
OCIToolCall,
|
||||
OCIToolDefinition,
|
||||
OCIVendors,
|
||||
)
|
||||
from litellm.llms.oci.common_utils import OCIError
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
def sha256_base64(data: bytes) -> str:
|
||||
digest = hashlib.sha256(data).digest()
|
||||
return base64.b64encode(digest).decode()
|
||||
|
||||
|
||||
def build_signature_string(method, path, headers, signed_headers):
|
||||
lines = []
|
||||
for header in signed_headers:
|
||||
if header == "(request-target)":
|
||||
value = f"{method.lower()} {path}"
|
||||
else:
|
||||
value = headers[header]
|
||||
lines.append(f"{header}: {value}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def load_private_key_from_str(key_str: str):
|
||||
key = serialization.load_pem_private_key(
|
||||
key_str.encode("utf-8"),
|
||||
password=None,
|
||||
)
|
||||
if not isinstance(key, rsa.RSAPrivateKey):
|
||||
raise TypeError(
|
||||
"The provided private key is not an RSA key, which is required for OCI signing."
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def get_vendor_from_model(model: str) -> OCIVendors:
|
||||
"""
|
||||
Extracts the vendor from the model name.
|
||||
Args:
|
||||
model (str): The model name.
|
||||
Returns:
|
||||
str: The vendor name.
|
||||
"""
|
||||
vendor = model.split(".")[0].lower()
|
||||
if vendor == "cohere":
|
||||
return OCIVendors.COHERE
|
||||
else:
|
||||
return OCIVendors.GENERIC
|
||||
|
||||
|
||||
# 5 minute timeout (models may need to load)
|
||||
STREAMING_TIMEOUT = 60 * 5
|
||||
|
||||
|
||||
class OCIChatConfig(BaseConfig):
|
||||
"""
|
||||
Configuration class for OCI's API interface.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
) -> None:
|
||||
locals_ = locals().copy()
|
||||
for key, value in locals_.items():
|
||||
if key != "self" and value is not None:
|
||||
setattr(self.__class__, key, value)
|
||||
# mark the class as using a custom stream wrapper because the default only iterates on lines
|
||||
setattr(self.__class__, "has_custom_stream_wrapper", True)
|
||||
|
||||
self.openai_to_oci_generic_param_map = {
|
||||
"stream": "isStream",
|
||||
"max_tokens": "maxTokens",
|
||||
"max_completion_tokens": "maxTokens",
|
||||
"temperature": "temperature",
|
||||
"tools": "tools",
|
||||
"frequency_penalty": "frequencyPenalty",
|
||||
"logprobs": "logProbs",
|
||||
"logit_bias": "logitBias",
|
||||
"n": "numGenerations",
|
||||
"presence_penalty": "presencePenalty",
|
||||
"seed": "seed",
|
||||
"stop": "stop",
|
||||
"tool_choice": "toolChoice",
|
||||
"top_p": "topP",
|
||||
"max_retries": False,
|
||||
"top_logprobs": False,
|
||||
"modalities": False,
|
||||
"prediction": False,
|
||||
"stream_options": False,
|
||||
"function_call": False,
|
||||
"functions": False,
|
||||
"extra_headers": False,
|
||||
"parallel_tool_calls": False,
|
||||
"audio": False,
|
||||
"web_search_options": False,
|
||||
}
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
supported_params = []
|
||||
vendor = get_vendor_from_model(model)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
for key, value in open_ai_to_oci_param_map.items():
|
||||
if value:
|
||||
supported_params.append(key)
|
||||
|
||||
return supported_params
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
adapted_params = {}
|
||||
vendor = get_vendor_from_model(model)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
|
||||
all_params = {**non_default_params, **optional_params}
|
||||
|
||||
for key, value in all_params.items():
|
||||
alias = open_ai_to_oci_param_map.get(key)
|
||||
|
||||
if alias is False:
|
||||
if drop_params:
|
||||
continue
|
||||
|
||||
raise Exception(f"param `{key}` is not supported on OCI")
|
||||
|
||||
if alias is None:
|
||||
adapted_params[key] = value
|
||||
continue
|
||||
|
||||
adapted_params[alias] = value
|
||||
|
||||
return adapted_params
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict,
|
||||
optional_params: dict,
|
||||
request_data: dict,
|
||||
api_base: str,
|
||||
api_key: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
stream: Optional[bool] = None,
|
||||
fake_stream: Optional[bool] = None,
|
||||
) -> Tuple[dict, Optional[bytes]]:
|
||||
"""
|
||||
Some providers like Bedrock require signing the request. The sign request funtion needs access to `request_data` and `complete_url`
|
||||
Args:
|
||||
headers: dict
|
||||
optional_params: dict
|
||||
request_data: dict - the request body being sent in http request
|
||||
api_base: str - the complete url being sent in http request
|
||||
Returns:
|
||||
dict - the signed headers
|
||||
"""
|
||||
import json
|
||||
|
||||
oci_region = optional_params.get("oci_region", "us-ashburn-1")
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
|
||||
)
|
||||
oci_user = optional_params.get("oci_user")
|
||||
oci_fingerprint = optional_params.get("oci_fingerprint")
|
||||
oci_tenancy = optional_params.get("oci_tenancy")
|
||||
oci_key = optional_params.get("oci_key")
|
||||
|
||||
if not oci_user or not oci_fingerprint or not oci_tenancy or not oci_key:
|
||||
raise Exception(
|
||||
"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key"
|
||||
)
|
||||
|
||||
method = str(optional_params.get("method", "POST")).upper()
|
||||
body = json.dumps(request_data).encode("utf-8")
|
||||
parsed = urlparse(api_base)
|
||||
path = parsed.path or "/"
|
||||
host = parsed.netloc
|
||||
|
||||
date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
|
||||
content_type = headers.get("content-type", "application/json")
|
||||
content_length = str(len(body))
|
||||
x_content_sha256 = sha256_base64(body)
|
||||
|
||||
headers_to_sign = {
|
||||
"date": date,
|
||||
"host": host,
|
||||
"content-type": content_type,
|
||||
"content-length": content_length,
|
||||
"x-content-sha256": x_content_sha256,
|
||||
}
|
||||
|
||||
signed_headers = [
|
||||
"date",
|
||||
"(request-target)",
|
||||
"host",
|
||||
"content-length",
|
||||
"content-type",
|
||||
"x-content-sha256",
|
||||
]
|
||||
signing_string = build_signature_string(
|
||||
method, path, headers_to_sign, signed_headers
|
||||
)
|
||||
|
||||
private_key = load_private_key_from_str(oci_key)
|
||||
signature = private_key.sign(
|
||||
signing_string.encode("utf-8"),
|
||||
padding.PKCS1v15(),
|
||||
hashes.SHA256(),
|
||||
)
|
||||
signature_b64 = base64.b64encode(signature).decode()
|
||||
|
||||
key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}"
|
||||
|
||||
authorization = (
|
||||
'Signature version="1",'
|
||||
f'keyId="{key_id}",'
|
||||
'algorithm="rsa-sha256",'
|
||||
f'headers="{" ".join(signed_headers)}",'
|
||||
f'signature="{signature_b64}"'
|
||||
)
|
||||
|
||||
headers.update(
|
||||
{
|
||||
"authorization": authorization,
|
||||
"date": date,
|
||||
"host": host,
|
||||
"content-type": content_type,
|
||||
"content-length": content_length,
|
||||
"x-content-sha256": x_content_sha256,
|
||||
}
|
||||
)
|
||||
|
||||
return headers, None
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
oci_region = optional_params.get("oci_region", "us-ashburn-1")
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
|
||||
)
|
||||
oci_user = optional_params.get("oci_user")
|
||||
oci_fingerprint = optional_params.get("oci_fingerprint")
|
||||
oci_tenancy = optional_params.get("oci_tenancy")
|
||||
oci_key = optional_params.get("oci_key")
|
||||
oci_compartment_id = optional_params.get("oci_compartment_id")
|
||||
|
||||
if (
|
||||
not oci_user
|
||||
or not oci_fingerprint
|
||||
or not oci_tenancy
|
||||
or not oci_key
|
||||
or not oci_compartment_id
|
||||
):
|
||||
raise Exception(
|
||||
"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id"
|
||||
)
|
||||
|
||||
if not api_base:
|
||||
raise Exception(
|
||||
"Either `api_base` must be provided or `litellm.api_base` must be set. Alternatively, you can set the `oci_region` optional parameter to use the default OCI region."
|
||||
)
|
||||
|
||||
headers.update(
|
||||
{
|
||||
"content-type": "application/json",
|
||||
"user-agent": f"litellm/{version}",
|
||||
}
|
||||
)
|
||||
|
||||
if not messages:
|
||||
raise Exception(
|
||||
"kwarg `messages` must be an array of messages that follow the openai chat standard"
|
||||
)
|
||||
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
oci_region = optional_params.get("oci_region", "us-ashburn-1")
|
||||
return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat"
|
||||
|
||||
def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict:
|
||||
selected_params = {}
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
|
||||
|
||||
for value in open_ai_to_oci_param_map.values():
|
||||
if value in optional_params:
|
||||
selected_params[value] = optional_params[value]
|
||||
if "tools" in selected_params:
|
||||
selected_params["tools"] = adapt_tool_definition_to_oci_standard(
|
||||
selected_params["tools"], vendor
|
||||
)
|
||||
return selected_params
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
oci_compartment_id = optional_params.get("oci_compartment_id", None)
|
||||
if not oci_compartment_id:
|
||||
raise Exception("kwarg `oci_compartment_id` is required for OCI requests")
|
||||
|
||||
vendor = get_vendor_from_model(model)
|
||||
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise Exception(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
data = OCICompletionPayload(
|
||||
compartmentId=oci_compartment_id,
|
||||
servingMode=OCIServingMode(
|
||||
servingType="ON_DEMAND",
|
||||
modelId=model,
|
||||
),
|
||||
chatRequest=OCIChatRequestPayload(
|
||||
apiFormat=vendor.value,
|
||||
messages=adapt_messages_to_generic_oci_standard(messages),
|
||||
**self._get_optional_params(vendor, optional_params),
|
||||
),
|
||||
)
|
||||
|
||||
return data.model_dump(exclude_none=True)
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ModelResponse:
|
||||
json = raw_response.json() # noqa: F811
|
||||
|
||||
error = json.get("error")
|
||||
|
||||
if error is not None:
|
||||
raise OCIError(
|
||||
message=str(json["error"]),
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
if not isinstance(json, dict):
|
||||
raise OCIError(
|
||||
message="Invalid response format from OCI",
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
try:
|
||||
completion_response = OCICompletionResponse(**json)
|
||||
except TypeError as e:
|
||||
raise OCIError(
|
||||
message=f"Response cannot be casted to OCICompletionResponse: {str(e)}",
|
||||
status_code=raw_response.status_code,
|
||||
)
|
||||
|
||||
vendor = get_vendor_from_model(model)
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
iso_str = completion_response.chatResponse.timeCreated
|
||||
dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
|
||||
model_response.created = int(dt.timestamp())
|
||||
|
||||
model_response.model = completion_response.modelId
|
||||
|
||||
message = model_response.choices[0].message # type: ignore
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
response_message = completion_response.chatResponse.choices[0].message
|
||||
if response_message.content and response_message.content[0].type == "TEXT":
|
||||
message.content = response_message.content[0].text
|
||||
if response_message.toolCalls:
|
||||
message.tool_calls = adapt_tools_to_openai_standard(
|
||||
response_message.toolCalls
|
||||
)
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=completion_response.chatResponse.usage.promptTokens,
|
||||
completion_tokens=completion_response.chatResponse.usage.completionTokens,
|
||||
total_tokens=completion_response.chatResponse.usage.totalTokens,
|
||||
)
|
||||
model_response.usage = usage # type: ignore
|
||||
|
||||
model_response._hidden_params["additional_headers"] = raw_response.headers
|
||||
|
||||
return model_response
|
||||
|
||||
@track_llm_api_timing()
|
||||
def get_sync_custom_stream_wrapper(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_base: str,
|
||||
headers: dict,
|
||||
data: dict,
|
||||
messages: list,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
signed_json_body: Optional[bytes] = None,
|
||||
) -> "OCIStreamWrapper":
|
||||
if "stream" in data:
|
||||
del data["stream"]
|
||||
if client is None or isinstance(client, AsyncHTTPHandler):
|
||||
client = _get_httpx_client(params={})
|
||||
|
||||
try:
|
||||
response = client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
stream=True,
|
||||
logging_obj=logging_obj,
|
||||
timeout=STREAMING_TIMEOUT,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OCIError(status_code=e.response.status_code, message=e.response.text)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise OCIError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream = response.iter_text()
|
||||
|
||||
streaming_response = OCIStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return streaming_response
|
||||
|
||||
@track_llm_api_timing()
|
||||
async def get_async_custom_stream_wrapper(
|
||||
self,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
api_base: str,
|
||||
headers: dict,
|
||||
data: dict,
|
||||
messages: list,
|
||||
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
signed_json_body: Optional[bytes] = None,
|
||||
) -> "OCIStreamWrapper":
|
||||
if "stream" in data:
|
||||
del data["stream"]
|
||||
|
||||
if client is None or isinstance(client, HTTPHandler):
|
||||
client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={})
|
||||
|
||||
try:
|
||||
response = await client.post(
|
||||
api_base,
|
||||
headers=headers,
|
||||
data=json.dumps(data),
|
||||
stream=True,
|
||||
logging_obj=logging_obj,
|
||||
timeout=STREAMING_TIMEOUT,
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise OCIError(status_code=e.response.status_code, message=e.response.text)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise OCIError(status_code=response.status_code, message=response.text)
|
||||
|
||||
completion_stream = response.aiter_text()
|
||||
|
||||
streaming_response = OCIStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return streaming_response
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
return OCIError(status_code=status_code, message=error_message)
|
||||
|
||||
|
||||
open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = {
|
||||
"system": "SYSTEM",
|
||||
"user": "USER",
|
||||
"assistant": "ASSISTANT",
|
||||
"tool": "TOOL",
|
||||
}
|
||||
|
||||
|
||||
def adapt_messages_to_generic_oci_standard_content_message(
|
||||
role: str, content: str | list
|
||||
) -> OCIMessage:
|
||||
new_content: list[OCIContentPartUnion] = []
|
||||
if isinstance(content, str):
|
||||
return OCIMessage(
|
||||
role=open_ai_to_generic_oci_role_map[role],
|
||||
content=[OCITextContentPart(text=content)],
|
||||
toolCalls=None,
|
||||
toolCallId=None,
|
||||
)
|
||||
|
||||
# content is a list of content items:
|
||||
# [
|
||||
# {"type": "text", "text": "Hello"},
|
||||
# {"type": "image_url", "image_url": "https://example.com/image.png"}
|
||||
# ]
|
||||
for content_item in content:
|
||||
if not isinstance(content_item, dict):
|
||||
raise Exception("Each content item must be a dictionary")
|
||||
|
||||
type = content_item.get("type")
|
||||
if not isinstance(type, str):
|
||||
raise Exception("Prop `type` is not a string")
|
||||
|
||||
if type not in ["text", "image_url"]:
|
||||
raise Exception(f"Prop `{type}` is not supported")
|
||||
|
||||
if type == "text":
|
||||
text = content_item.get("text")
|
||||
if not isinstance(text, str):
|
||||
raise Exception("Prop `text` is not a string")
|
||||
new_content.append(OCITextContentPart(text=text))
|
||||
|
||||
elif type == "image_url":
|
||||
image_url = content_item.get("image_url")
|
||||
if not isinstance(image_url, str):
|
||||
raise Exception("Prop `image_url` is not a string")
|
||||
new_content.append(OCIImageContentPart(imageUrl=image_url))
|
||||
|
||||
return OCIMessage(
|
||||
role=open_ai_to_generic_oci_role_map[role],
|
||||
content=new_content,
|
||||
toolCalls=None,
|
||||
toolCallId=None,
|
||||
)
|
||||
|
||||
|
||||
def adapt_messages_to_generic_oci_standard_tool_call(
|
||||
role: str, tool_calls: list
|
||||
) -> OCIMessage:
|
||||
tool_calls_formated = []
|
||||
for tool_call in tool_calls:
|
||||
if not isinstance(tool_call, dict):
|
||||
raise Exception("Each tool call must be a dictionary")
|
||||
|
||||
if tool_call.get("type") != "function":
|
||||
raise Exception("OCI only supports function tools")
|
||||
|
||||
tool_call_id = tool_call.get("id")
|
||||
if not isinstance(tool_call_id, str):
|
||||
raise Exception("Prop `id` is not a string")
|
||||
|
||||
tool_function = tool_call.get("function")
|
||||
if not isinstance(tool_function, dict):
|
||||
raise Exception("Prop `function` is not a dictionary")
|
||||
|
||||
function_name = tool_function.get("name")
|
||||
if not isinstance(function_name, str):
|
||||
raise Exception("Prop `name` is not a string")
|
||||
|
||||
arguments = tool_call["function"].get("arguments", "{}")
|
||||
if not isinstance(arguments, str):
|
||||
raise Exception("Prop `arguments` is not a string")
|
||||
|
||||
# tool_calls_formated.append(OCIToolCall(
|
||||
# id=tool_call_id,
|
||||
# type="FUNCTION",
|
||||
# function=OCIFunction(
|
||||
# name=function_name,
|
||||
# arguments=arguments
|
||||
# )
|
||||
# ))
|
||||
|
||||
tool_calls_formated.append(
|
||||
OCIToolCall(
|
||||
id=tool_call_id,
|
||||
type="FUNCTION",
|
||||
name=function_name,
|
||||
arguments=arguments,
|
||||
)
|
||||
)
|
||||
|
||||
return OCIMessage(
|
||||
role=open_ai_to_generic_oci_role_map[role],
|
||||
content=None,
|
||||
toolCalls=tool_calls_formated,
|
||||
toolCallId=None,
|
||||
)
|
||||
|
||||
|
||||
def adapt_messages_to_generic_oci_standard_tool_response(
|
||||
role: str, tool_call_id: str, content: str
|
||||
) -> OCIMessage:
|
||||
return OCIMessage(
|
||||
role=open_ai_to_generic_oci_role_map[role],
|
||||
content=[OCITextContentPart(text=content)],
|
||||
toolCalls=None,
|
||||
toolCallId=tool_call_id,
|
||||
)
|
||||
|
||||
|
||||
def adapt_messages_to_generic_oci_standard(
|
||||
messages: List[AllMessageValues],
|
||||
) -> List[OCIMessage]:
|
||||
new_messages = []
|
||||
for message in messages:
|
||||
role = message["role"]
|
||||
content = message.get("content")
|
||||
tool_calls = message.get("tool_calls")
|
||||
tool_call_id = message.get("tool_call_id")
|
||||
|
||||
if role in ["system", "user", "assistant"] and content is not None:
|
||||
if not isinstance(content, (str, list)):
|
||||
raise Exception(
|
||||
"Prop `content` must be a string or a list of content items"
|
||||
)
|
||||
new_messages.append(
|
||||
adapt_messages_to_generic_oci_standard_content_message(role, content)
|
||||
)
|
||||
|
||||
elif role == "assistant" and tool_calls is not None:
|
||||
if not isinstance(tool_calls, list):
|
||||
raise Exception("Prop `tool_calls` must be a list of tool calls")
|
||||
new_messages.append(
|
||||
adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls)
|
||||
)
|
||||
|
||||
elif role == "tool":
|
||||
if not isinstance(tool_call_id, str):
|
||||
raise Exception("Prop `tool_call_id` is required and must be a string")
|
||||
if not isinstance(content, str):
|
||||
raise Exception("Prop `content` is not a string")
|
||||
new_messages.append(
|
||||
adapt_messages_to_generic_oci_standard_tool_response(
|
||||
role, tool_call_id, content
|
||||
)
|
||||
)
|
||||
|
||||
return new_messages
|
||||
|
||||
|
||||
def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors):
|
||||
new_tools = []
|
||||
if vendor == OCIVendors.COHERE:
|
||||
raise ValueError(
|
||||
"Cohere models are not yet supported in the litellm OCI chat completion endpoint. Use the Cohere API directly."
|
||||
)
|
||||
else:
|
||||
for tool in tools:
|
||||
if tool["type"] != "function":
|
||||
raise Exception("OCI only supports function tools")
|
||||
|
||||
tool_function = tool.get("function")
|
||||
if not isinstance(tool_function, dict):
|
||||
raise Exception("Prop `function` is not a dictionary")
|
||||
|
||||
new_tool = OCIToolDefinition(
|
||||
type="FUNCTION",
|
||||
name=tool_function.get("name"),
|
||||
description=tool_function.get("description", ""),
|
||||
parameters=tool_function.get("parameters", {}),
|
||||
)
|
||||
new_tools.append(new_tool)
|
||||
|
||||
return new_tools
|
||||
|
||||
|
||||
def adapt_tools_to_openai_standard(
|
||||
tools: list[OCIToolCall],
|
||||
) -> list[ChatCompletionMessageToolCall]:
|
||||
new_tools = []
|
||||
for tool in tools:
|
||||
new_tool = ChatCompletionMessageToolCall(
|
||||
id=tool.id,
|
||||
type="function",
|
||||
function={
|
||||
"name": tool.name,
|
||||
"arguments": tool.arguments,
|
||||
},
|
||||
)
|
||||
new_tools.append(new_tool)
|
||||
return new_tools
|
||||
|
||||
|
||||
class OCIStreamWrapper(CustomStreamWrapper):
|
||||
"""
|
||||
Custom stream wrapper for OCI responses.
|
||||
This class is used to handle streaming responses from OCI's API.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def chunk_creator(self, chunk: Any):
|
||||
if not isinstance(chunk, str):
|
||||
raise ValueError(f"Chunk is not a string: {chunk}")
|
||||
if not chunk.startswith("data:"):
|
||||
raise ValueError(f"Chunk does not start with 'data:': {chunk}")
|
||||
dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON
|
||||
try:
|
||||
typed_chunk = OCIStreamChunk(**dict_chunk)
|
||||
except TypeError as e:
|
||||
raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}")
|
||||
|
||||
if typed_chunk.index is None:
|
||||
typed_chunk.index = 0
|
||||
|
||||
text = ""
|
||||
if typed_chunk.message and typed_chunk.message.content:
|
||||
for item in typed_chunk.message.content:
|
||||
if isinstance(item, OCITextContentPart):
|
||||
text += item.text
|
||||
elif isinstance(item, OCIImageContentPart):
|
||||
raise ValueError(
|
||||
"OCI does not support image content in streaming responses"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported content type in OCI response: {item.type}"
|
||||
)
|
||||
|
||||
tool_calls = None
|
||||
if typed_chunk.message and typed_chunk.message.toolCalls:
|
||||
tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls)
|
||||
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=typed_chunk.index if typed_chunk.index else 0,
|
||||
delta=Delta(
|
||||
content=text,
|
||||
tool_calls=[tool.model_dump() for tool in tool_calls]
|
||||
if tool_calls
|
||||
else None,
|
||||
provider_specific_fields=None, # OCI does not have provider specific fields in the response
|
||||
thinking_blocks=None, # OCI does not have thinking blocks in the response
|
||||
reasoning_content=None, # OCI does not have reasoning content in the response
|
||||
),
|
||||
finish_reason=typed_chunk.finishReason,
|
||||
)
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
||||
class OCIError(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
message: str,
|
||||
headers: Optional[httpx.Headers] = None,
|
||||
):
|
||||
super().__init__(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
headers=headers,
|
||||
)
|
||||
@@ -151,6 +151,7 @@ from .llms.gemini.common_utils import get_api_key_from_env
|
||||
from .llms.groq.chat.handler import GroqChatCompletion
|
||||
from .llms.huggingface.embedding.handler import HuggingFaceEmbedding
|
||||
from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
|
||||
from .llms.oci.chat.transformation import OCIChatConfig
|
||||
from .llms.ollama.completion import handler as ollama
|
||||
from .llms.oobabooga.chat import oobabooga
|
||||
from .llms.openai.completion.handler import OpenAITextCompletion
|
||||
@@ -252,6 +253,7 @@ base_llm_http_handler = BaseLLMHTTPHandler()
|
||||
base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler()
|
||||
sagemaker_chat_completion = SagemakerChatHandler()
|
||||
bytez_transformation = BytezChatConfig()
|
||||
oci_transformation = OCIChatConfig()
|
||||
####### COMPLETION ENDPOINTS ################
|
||||
|
||||
|
||||
@@ -2399,6 +2401,24 @@ def completion( # type: ignore # noqa: PLR0915
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
)
|
||||
elif custom_llm_provider == "oci":
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
timeout=timeout, # type: ignore
|
||||
client=client,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
encoding=encoding,
|
||||
stream=stream,
|
||||
)
|
||||
elif custom_llm_provider == "oobabooga":
|
||||
custom_llm_provider = "oobabooga"
|
||||
model_response = oobabooga.completion(
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
from enum import Enum
|
||||
|
||||
OCIRoles = Literal["SYSTEM", "USER", "ASSISTANT", "TOOL"]
|
||||
|
||||
|
||||
class OCIVendors(Enum):
|
||||
"""
|
||||
A class to hold the vendor names for OCI models.
|
||||
This is used to map model names to their respective vendors.
|
||||
"""
|
||||
|
||||
COHERE = "COHERE"
|
||||
GENERIC = "GENERIC"
|
||||
|
||||
|
||||
# --- Base Models and Content Parts ---
|
||||
|
||||
|
||||
class OCIContentPart(BaseModel):
|
||||
"""Base model for content parts in an OCI message."""
|
||||
|
||||
type: str
|
||||
|
||||
|
||||
class OCITextContentPart(OCIContentPart):
|
||||
"""Text content part for the OCI API."""
|
||||
|
||||
type: Literal["TEXT"] = "TEXT"
|
||||
text: str
|
||||
|
||||
|
||||
class OCIImageContentPart(OCIContentPart):
|
||||
"""Image content part for the OCI API."""
|
||||
|
||||
type: Literal["IMAGE"] = "IMAGE"
|
||||
imageUrl: str
|
||||
|
||||
|
||||
OCIContentPartUnion = Union[OCITextContentPart, OCIImageContentPart]
|
||||
|
||||
# --- Models for Tools and Tool Calls ---
|
||||
|
||||
|
||||
class OCIToolCall(BaseModel):
|
||||
"""Represents a tool call made by the model."""
|
||||
|
||||
id: str
|
||||
type: Literal["FUNCTION"] = "FUNCTION"
|
||||
name: str
|
||||
arguments: str # Arguments should be a JSON-serialized string
|
||||
|
||||
|
||||
class OCIToolDefinition(BaseModel):
|
||||
"""Defines a tool that can be used by the model."""
|
||||
|
||||
type: Literal["FUNCTION"] = "FUNCTION"
|
||||
name: str | None = None
|
||||
description: str | None = None
|
||||
parameters: dict | None = None
|
||||
|
||||
|
||||
# --- Message Models (Request and Response) ---
|
||||
|
||||
|
||||
class OCIMessage(BaseModel):
|
||||
"""Model for a single message in the request/response payload."""
|
||||
|
||||
role: OCIRoles
|
||||
content: list[OCIContentPartUnion] | None = None
|
||||
toolCalls: list[OCIToolCall] | None = None
|
||||
toolCallId: str | None = None
|
||||
|
||||
|
||||
# --- Request Payload Models ---
|
||||
|
||||
|
||||
class OCIChatRequestPayload(BaseModel):
|
||||
"""Internal 'chatRequest' payload for the OCI API."""
|
||||
|
||||
apiFormat: str
|
||||
messages: list[OCIMessage]
|
||||
tools: list[OCIToolDefinition] | None = None
|
||||
isStream: bool = False
|
||||
numGenerations: int | None = None
|
||||
maxTokens: int | None = None
|
||||
temperature: float | None = None
|
||||
topP: float | None = None
|
||||
stop: list[str] | None = None
|
||||
seed: int | None = None
|
||||
frequencyPenalty: float | None = None
|
||||
presencePenalty: float | None = None
|
||||
|
||||
|
||||
class OCIServingMode(BaseModel):
|
||||
"""Defines the serving mode and the model to be used."""
|
||||
|
||||
servingType: str
|
||||
modelId: str
|
||||
|
||||
|
||||
class OCICompletionPayload(BaseModel):
|
||||
"""Pydantic model for the complete OCI chat request body."""
|
||||
|
||||
compartmentId: str
|
||||
servingMode: OCIServingMode
|
||||
chatRequest: OCIChatRequestPayload
|
||||
|
||||
|
||||
# --- API Response Models (Non-streaming) ---
|
||||
|
||||
|
||||
class OCICompletionTokenDetails(BaseModel):
|
||||
"""Completion token details in the OCI response."""
|
||||
|
||||
acceptedPredictionTokens: int
|
||||
reasoningTokens: int
|
||||
|
||||
|
||||
class OCIPropmtTokensDetails(BaseModel):
|
||||
"""Prompt token details in the OCI response."""
|
||||
|
||||
cachedTokens: int
|
||||
|
||||
|
||||
class OCIResponseUsage(BaseModel):
|
||||
"""Token usage in the OCI response."""
|
||||
|
||||
promptTokens: int
|
||||
completionTokens: int
|
||||
totalTokens: int
|
||||
completionTokensDetails: OCICompletionTokenDetails
|
||||
promptTokensDetails: OCIPropmtTokensDetails
|
||||
|
||||
|
||||
class OCIResponseChoice(BaseModel):
|
||||
"""A completion choice in the OCI response."""
|
||||
|
||||
index: int
|
||||
message: OCIMessage
|
||||
finishReason: str | None
|
||||
logprobs: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class OCIChatResponse(BaseModel):
|
||||
"""The 'chatResponse' object in the OCI response."""
|
||||
|
||||
apiFormat: str
|
||||
timeCreated: str
|
||||
choices: list[OCIResponseChoice]
|
||||
usage: OCIResponseUsage
|
||||
|
||||
|
||||
class OCICompletionResponse(BaseModel):
|
||||
"""Model for the complete non-streaming OCI response body."""
|
||||
|
||||
modelId: str
|
||||
modelVersion: str
|
||||
chatResponse: OCIChatResponse
|
||||
|
||||
|
||||
# --- API Response Models (Streaming) ---
|
||||
|
||||
|
||||
class OCIStreamDelta(BaseModel):
|
||||
"""The content delta in a streaming chunk."""
|
||||
|
||||
content: list[OCIContentPartUnion] | None = None
|
||||
role: str | None = None
|
||||
toolCalls: list[OCIToolCall] | None = None
|
||||
|
||||
|
||||
class OCIStreamChunk(BaseModel):
|
||||
"""Model for a single SSE event chunk from OCI."""
|
||||
|
||||
finishReason: str | None = None
|
||||
message: OCIStreamDelta | None = None
|
||||
pad: str | None = None
|
||||
index: int | None = None
|
||||
@@ -2321,6 +2321,7 @@ class LlmProviders(str, Enum):
|
||||
PG_VECTOR = "pg_vector"
|
||||
HYPERBOLIC = "hyperbolic"
|
||||
RECRAFT = "recraft"
|
||||
OCI = "oci"
|
||||
AUTO_ROUTER = "auto_router"
|
||||
DOTPROMPT = "dotprompt"
|
||||
|
||||
|
||||
@@ -6916,6 +6916,8 @@ class ProviderConfigManager:
|
||||
return litellm.OpenAIGPTConfig()
|
||||
elif litellm.LlmProviders.NSCALE == provider:
|
||||
return litellm.NscaleConfig()
|
||||
elif litellm.LlmProviders.OCI == provider:
|
||||
return litellm.OCIChatConfig()
|
||||
elif litellm.LlmProviders.HYPERBOLIC == provider:
|
||||
return litellm.HyperbolicChatConfig()
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import httpx
|
||||
import pytest
|
||||
import json
|
||||
|
||||
import litellm
|
||||
|
||||
# Adds the parent directory to the system path
|
||||
sys.path.insert(0, os.path.abspath("../../../../.."))
|
||||
|
||||
from litellm import ModelResponse
|
||||
from litellm.llms.oci.chat.transformation import OCIChatConfig, version
|
||||
|
||||
TEST_MODEL_NAME = "xai.grok-4"
|
||||
TEST_MODEL = f"oci/{TEST_MODEL_NAME}"
|
||||
TEST_MESSAGES = [{"role": "user", "content": "Hello, how are you?"}]
|
||||
TEST_COMPARTMENT_ID = "ocid1.compartment.oc1..xxxxxx"
|
||||
TEST_OCI_PARAMS = {
|
||||
"oci_region": "us-ashburn-1",
|
||||
"oci_user": "ocid1.user.oc1..xxxxxxEXAMPLExxxxxx",
|
||||
"oci_fingerprint": "4f:29:77:cc:b1:3e:55:ab:61:2a:de:47:f1:38:4c:90",
|
||||
"oci_tenancy": "ocid1.tenancy.oc1..xxxxxxEXAMPLExxxxxx",
|
||||
"oci_compartment_id": TEST_COMPARTMENT_ID,
|
||||
"oci_key": "<private_key.pem as string>"
|
||||
}
|
||||
|
||||
|
||||
class TestOCIChatConfig:
|
||||
def test_validate_environment_with_oci_region(self):
|
||||
config = OCIChatConfig()
|
||||
|
||||
headers = {}
|
||||
|
||||
result = config.validate_environment(
|
||||
headers=headers,
|
||||
model=TEST_MODEL,
|
||||
messages=TEST_MESSAGES, # type: ignore
|
||||
optional_params=TEST_OCI_PARAMS,
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert result["content-type"] == "application/json"
|
||||
assert result["user-agent"] == f"litellm/{version}"
|
||||
|
||||
def test_missing_oci_auth_parameters(self):
|
||||
optional_params = TEST_OCI_PARAMS.copy()
|
||||
optional_params.pop("oci_region")
|
||||
|
||||
# Remove optional_params one by one and verify that an exception is raised
|
||||
for key in optional_params.keys():
|
||||
modified_params = optional_params.copy()
|
||||
del modified_params[key]
|
||||
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
config = OCIChatConfig()
|
||||
|
||||
headers = {}
|
||||
|
||||
config.validate_environment(
|
||||
headers=headers,
|
||||
model=TEST_MODEL,
|
||||
messages=TEST_MESSAGES, # type: ignore
|
||||
optional_params=modified_params,
|
||||
api_base="https://api.oci.example.com",
|
||||
litellm_params={},
|
||||
)
|
||||
|
||||
assert f"Missing one of the following parameters: oci_user, oci_fingerprint, oci_tenancy, oci_key, oci_compartment_id" in str(excinfo.value)
|
||||
|
||||
def test_transform_request_simple(self):
|
||||
"""
|
||||
Tests if a simple request is transformed correctly.
|
||||
"""
|
||||
config = OCIChatConfig()
|
||||
optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID}
|
||||
transformed_request = config.transform_request(
|
||||
model=TEST_MODEL_NAME,
|
||||
messages=TEST_MESSAGES, # type: ignore
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
expected_output = {
|
||||
"compartmentId": TEST_COMPARTMENT_ID,
|
||||
"servingMode": {"servingType": "ON_DEMAND", "modelId": TEST_MODEL_NAME},
|
||||
"chatRequest": {
|
||||
"apiFormat": "GENERIC",
|
||||
"isStream": False,
|
||||
"messages": [
|
||||
{
|
||||
"role": "USER",
|
||||
"content": [{"type": "TEXT", "text": "Hello, how are you?"}],
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
assert transformed_request == expected_output
|
||||
|
||||
def test_transform_request_with_tools(self):
|
||||
"""
|
||||
Tests if a request with tools is transformed correctly.
|
||||
"""
|
||||
config = OCIChatConfig()
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
optional_params = {
|
||||
"oci_compartment_id": TEST_COMPARTMENT_ID,
|
||||
"tools": tools,
|
||||
}
|
||||
transformed_request = config.transform_request(
|
||||
model=TEST_MODEL_NAME,
|
||||
messages=TEST_MESSAGES, # type: ignore
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "tools" in transformed_request["chatRequest"]
|
||||
assert transformed_request["chatRequest"]["tools"][0]["name"] == "get_current_weather"
|
||||
assert transformed_request["chatRequest"]["tools"][0]["type"] == "FUNCTION"
|
||||
assert transformed_request["chatRequest"]["tools"][0]["description"] == "Get the current weather in a given location"
|
||||
assert transformed_request["chatRequest"]["tools"][0]["parameters"] is not None
|
||||
|
||||
def test_transform_response_simple_text(self):
|
||||
"""
|
||||
Tests if a simple text response is transformed correctly.
|
||||
"""
|
||||
config = OCIChatConfig()
|
||||
created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
mock_oci_response = {
|
||||
"modelId": TEST_MODEL_NAME,
|
||||
"modelVersion": "1.0",
|
||||
"chatResponse": {
|
||||
"apiFormat": "GENERIC",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "ASSISTANT",
|
||||
"content": [{"type": "TEXT", "text": "I am doing well, thank you!"}],
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
}
|
||||
],
|
||||
"timeCreated": created_time,
|
||||
"usage": {
|
||||
"promptTokens": 10,
|
||||
"completionTokens": 20,
|
||||
"totalTokens": 30,
|
||||
"completionTokensDetails": {
|
||||
"acceptedPredictionTokens": 20,
|
||||
"reasoningTokens": 20,
|
||||
},
|
||||
"promptTokensDetails": {
|
||||
"cachedTokens": 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
response = httpx.Response(
|
||||
status_code=200, json=mock_oci_response, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
result = config.transform_response(
|
||||
model=TEST_MODEL_NAME,
|
||||
raw_response=response,
|
||||
model_response=ModelResponse(),
|
||||
logging_obj={}, # type: ignore
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding={},
|
||||
)
|
||||
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert len(result.choices) == 1
|
||||
assert isinstance(result.choices[0], litellm.Choices)
|
||||
assert result.choices[0].message
|
||||
assert result.choices[0].message.content == "I am doing well, thank you!"
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
assert result.model == TEST_MODEL_NAME
|
||||
assert hasattr(result, "usage")
|
||||
assert isinstance(result.usage, litellm.Usage) # type: ignore
|
||||
assert result.usage.prompt_tokens == 10 # type: ignore
|
||||
assert result.usage.completion_tokens == 20 # type: ignore
|
||||
assert result.usage.total_tokens == 30 # type: ignore
|
||||
|
||||
def test_transform_response_with_tool_calls(self):
|
||||
"""
|
||||
Tests if a response with tool calls is transformed correctly.
|
||||
"""
|
||||
config = OCIChatConfig()
|
||||
created_time = datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
mock_oci_response = {
|
||||
"modelId": TEST_MODEL_NAME,
|
||||
"modelVersion": "1.0",
|
||||
"chatResponse": {
|
||||
"apiFormat": "GENERIC",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "ASSISTANT",
|
||||
"content": None,
|
||||
"toolCalls": [
|
||||
{
|
||||
"id": "call_abc123",
|
||||
"type": "FUNCTION",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"location": "Vila Velha, BR"}',
|
||||
}
|
||||
],
|
||||
},
|
||||
"finishReason": "stop",
|
||||
}
|
||||
],
|
||||
"timeCreated": created_time,
|
||||
"usage": {
|
||||
"promptTokens": 10,
|
||||
"completionTokens": 20,
|
||||
"totalTokens": 30,
|
||||
"completionTokensDetails": {
|
||||
"acceptedPredictionTokens": 20,
|
||||
"reasoningTokens": 20,
|
||||
},
|
||||
"promptTokensDetails": {
|
||||
"cachedTokens": 10,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
response = httpx.Response(status_code=200, json=mock_oci_response)
|
||||
model_response = ModelResponse(
|
||||
choices=[litellm.Choices(index=0, message=litellm.Message())]
|
||||
)
|
||||
|
||||
result = config.transform_response(
|
||||
model=TEST_MODEL_NAME,
|
||||
raw_response=response,
|
||||
model_response=model_response,
|
||||
logging_obj={}, # type: ignore
|
||||
request_data={},
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding={},
|
||||
)
|
||||
|
||||
# General assertions
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert len(result.choices) == 1
|
||||
|
||||
choice = result.choices[0]
|
||||
assert isinstance(choice, litellm.Choices)
|
||||
assert choice.finish_reason == "stop"
|
||||
|
||||
# Message and tool_calls assertions
|
||||
message = choice.message
|
||||
assert isinstance(message, litellm.Message)
|
||||
assert hasattr(message, "tool_calls")
|
||||
assert isinstance(message.tool_calls, list)
|
||||
assert len(message.tool_calls) == 1
|
||||
|
||||
# Specific tool_call assertions
|
||||
tool_call = message.tool_calls[0]
|
||||
assert isinstance(tool_call, litellm.utils.ChatCompletionMessageToolCall)
|
||||
assert tool_call.id == "call_abc123"
|
||||
assert tool_call.type == "function"
|
||||
assert tool_call.function["name"] == "get_weather"
|
||||
assert tool_call.function["arguments"] == '{"location": "Vila Velha, BR"}'
|
||||
|
||||
# Usage assertions
|
||||
assert hasattr(result, "usage")
|
||||
usage = result.usage # type: ignore
|
||||
assert isinstance(usage, litellm.Usage) # type: ignore
|
||||
assert usage.prompt_tokens == 10 # type: ignore
|
||||
assert usage.completion_tokens == 20 # type: ignore
|
||||
assert usage.total_tokens == 30 # type: ignore
|
||||
Reference in New Issue
Block a user