diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e86fca17c7..babe3b6293 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -53,3 +53,31 @@ jobs: uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 with: category: "/language:${{ matrix.language }}" + output: sarif-results + upload: failure-only + + # py/weak-sensitive-data-hashing (CWE-328) fires on the OCI signing call at + # litellm/llms/oci/common_utils.py, which hashes the HTTP request body to + # produce the x-content-sha256 header required by the OCI HTTP signing spec — + # a content-integrity hash, not a password or secret hash. SHA-256 is mandated + # by Oracle for this header; see + # https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm + # The `usedforsecurity=False` flag on the hashlib.sha256 call already declares + # non-security intent, but CodeQL's taint flow still re-fires when callers + # further up the stack are modified. The suppression is scoped to this one + # file/rule pair via SARIF post-filtering so every other callsite of + # py/weak-sensitive-data-hashing in the repository continues to be analyzed. + - name: Filter SARIF (OCI sha256) + if: matrix.language == 'python' + uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1 + with: + patterns: | + -litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing + input: sarif-results/python.sarif + output: sarif-results/python.sarif + + - name: Upload SARIF + uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3 + with: + sarif_file: sarif-results + category: "/language:${{ matrix.language }}" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 96fdf4494f..090aac187b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -890,6 +890,18 @@ class BaseLLMHTTPHandler: headers=headers, ) + # Some providers (e.g. OCI) require request signing after the body is built. + # The default BaseConfig.sign_request returns (headers, None) — a no-op for + # providers that don't need signing. + headers, signed_body = provider_config.sign_request( + headers=headers, + optional_params=optional_params, + request_data=data, + api_base=api_base, + api_key=api_key, + model=model, + ) + ## LOGGING logging_obj.pre_call( input=input, @@ -916,6 +928,7 @@ class BaseLLMHTTPHandler: client=client, optional_params=optional_params, litellm_params=litellm_params, + signed_body=signed_body, ) if client is None or not isinstance(client, HTTPHandler): @@ -926,12 +939,20 @@ class BaseLLMHTTPHandler: sync_httpx_client = client try: - response = sync_httpx_client.post( - url=api_base, - headers=headers, - data=json.dumps(data), - timeout=timeout, - ) + if signed_body is not None: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=signed_body, + timeout=timeout, + ) + else: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=json.dumps(data), + timeout=timeout, + ) except Exception as e: raise self._handle_error( e=e, @@ -964,6 +985,7 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + signed_body: Optional[bytes] = None, ) -> EmbeddingResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -974,12 +996,20 @@ class BaseLLMHTTPHandler: async_httpx_client = client try: - response = await async_httpx_client.post( - url=api_base, - headers=headers, - json=request_data, - timeout=timeout, - ) + if signed_body is not None: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=signed_body, + timeout=timeout, + ) + else: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=request_data, + timeout=timeout, + ) except Exception as e: raise self._handle_error(e=e, provider_config=provider_config) diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py new file mode 100644 index 0000000000..ac92fd22aa --- /dev/null +++ b/litellm/llms/oci/chat/cohere.py @@ -0,0 +1,386 @@ +""" +OCI Generative AI — Cohere-specific chat transformation helpers. + +Handles message history building, tool definition adaptation, non-streaming +response parsing, and streaming chunk parsing for models served with +``apiFormat="COHERE"`` (e.g. ``cohere.command-*``). +""" + +import datetime +import json +from typing import Any, Dict, List, Optional + +import httpx +from pydantic import ValidationError + +from litellm.llms.oci.chat.generic import ( + _normalize_oci_finish_reason, + _synthesize_oci_tool_call_id, +) +from litellm.llms.oci.common_utils import ( + OCI_JSON_TO_PYTHON_TYPES, + OCIError, + enrich_cohere_param_description, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, +) +from litellm.types.llms.oci import ( + CohereChatResult, + CohereMessage, + CohereParameterDefinition, + CohereStreamChunk, + CohereTool, + CohereToolCall, + CohereToolMessage, + CohereToolResult, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Choices, + Delta, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) +from litellm.types.utils import Usage + + +def _extract_text_content(content: Any) -> str: + """Return the plain-text representation of a message content value.""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + item.get("text", "") + for item in content + if isinstance(item, dict) and item.get("type") == "text" + ) + return str(content) + + +def adapt_messages_to_cohere_standard( + messages: List[AllMessageValues], +) -> List[CohereMessage]: + """Build a Cohere ``chatHistory`` list from an OpenAI-format message array. + + - All messages except the *last user message* are included. The caller pulls + the last user message into the request's top-level ``message`` field, so + trailing tool results (the standard agentic continuation pattern) still + appear in ``chatHistory`` and reach the model. + - If no user message exists, every message is included (no slice). + - System messages must be filtered out by the caller (they are routed into + ``preambleOverride`` separately) — they are not represented in + ``chatHistory``. + - Tool results are expressed as OCI ``CohereToolMessage.toolResults`` entries, + with the originating call's name and parameters resolved from the preceding + assistant message via a ``tool_call_id`` lookup. + """ + # First pass: build tool_call_id → CohereToolCall so tool-result messages can + # reference the originating call by name and parameters. + tool_call_lookup: Dict[str, CohereToolCall] = {} + for msg in messages: + if msg.get("role") == "assistant": + tool_calls_raw: Any = msg.get("tool_calls") or [] + for tc in tool_calls_raw: + tc_id = tc.get("id", "") + raw_args: Any = tc.get("function", {}).get("arguments", "{}") + try: + params: Dict[str, Any] = ( + json.loads(raw_args) if isinstance(raw_args, str) else raw_args + ) + except json.JSONDecodeError: + params = {} + tool_call_lookup[tc_id] = CohereToolCall( + name=str(tc.get("function", {}).get("name", "")), + parameters=params, + ) + + last_user_index = next( + ( + i + for i in range(len(messages) - 1, -1, -1) + if messages[i].get("role") == "user" + ), + None, + ) + history_source = ( + messages + if last_user_index is None + else [m for i, m in enumerate(messages) if i != last_user_index] + ) + + chat_history: List[CohereMessage] = [] + for msg in history_source: + role = msg.get("role") + content = _extract_text_content(msg.get("content")) + + tool_calls: Optional[List[CohereToolCall]] = None + if role == "assistant" and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] + tool_calls = [] + for tc in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item] + raw_arguments: Any = tc.get("function", {}).get("arguments", {}) + if isinstance(raw_arguments, str): + try: + arguments: Dict[str, Any] = json.loads(raw_arguments) + except json.JSONDecodeError: + arguments = {} + else: + arguments = raw_arguments + tool_calls.append( + CohereToolCall( + name=str(tc.get("function", {}).get("name", "")), + parameters=arguments, + ) + ) + + if role == "user": + chat_history.append(CohereMessage(role="USER", message=content)) + elif role == "assistant": + chat_history.append( + CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls) + ) + elif role == "tool": + tool_call_id = str(msg.get("tool_call_id", "") or "") + cohere_call = tool_call_lookup.get( + tool_call_id, CohereToolCall(name="", parameters={}) + ) + tool_result = CohereToolResult( + call=cohere_call, + outputs=[{"output": content}], + ) + # OpenAI emits one tool-role message per parallel tool call, but + # the OCI Cohere API expects all results from a single assistant + # turn to share one TOOL history entry with multiple toolResults. + # Merge consecutive tool messages so the model sees the parallel + # call/result pairing correctly during agentic loops. + if chat_history and isinstance(chat_history[-1], CohereToolMessage): + chat_history[-1].toolResults.append(tool_result) + else: + chat_history.append(CohereToolMessage(toolResults=[tool_result])) + + return chat_history + + +def adapt_tool_definitions_to_cohere_standard( + tools: List[Dict[str, Any]], +) -> List[CohereTool]: + """Adapt OpenAI-format tool definitions to the OCI Cohere format. + + - Resolves ``$ref``/``$defs`` and ``anyOf`` patterns that OCI rejects. + - Maps JSON Schema type names to Python type names (``"string"`` → ``"str"``). + - Embeds unsupported constraints (enum, format, range, pattern) into the + parameter description so the model can still see them. + """ + cohere_tools = [] + for tool in tools: + function_def = tool.get("function", {}) + raw_params = function_def.get("parameters", {}) + + resolved = sanitize_oci_schema( + resolve_oci_schema_anyof(resolve_oci_schema_refs(raw_params)) + ) + properties = resolved.get("properties", {}) + required = resolved.get("required", []) + + parameter_definitions = {} + for param_name, param_schema in properties.items(): + json_type = param_schema.get("type", "string") + python_type = OCI_JSON_TO_PYTHON_TYPES.get(json_type, json_type) + parameter_definitions[param_name] = CohereParameterDefinition( + description=enrich_cohere_param_description( + param_schema.get("description", ""), param_schema + ), + type=python_type, + isRequired=param_name in required, + ) + + cohere_tools.append( + CohereTool( + name=function_def.get("name", ""), + description=function_def.get("description", ""), + parameterDefinitions=parameter_definitions, + ) + ) + + return cohere_tools + + +def handle_cohere_response( + json_response: dict, + model: str, + model_response: ModelResponse, + raw_response: httpx.Response, +) -> ModelResponse: + """Parse a non-streaming Cohere OCI response into a LiteLLM ModelResponse.""" + try: + cohere_response = CohereChatResult(**json_response) + except (TypeError, ValidationError) as e: + raise OCIError( + message=f"Response cannot be casted to CohereChatResult: {str(e)}", + status_code=raw_response.status_code, + ) + + model_response.model = model + model_response.created = int(datetime.datetime.now().timestamp()) + + response_text = cohere_response.chatResponse.text + finish_reason = _normalize_oci_finish_reason( + cohere_response.chatResponse.finishReason + ) + + tool_calls: Optional[List[Dict[str, Any]]] = None + if cohere_response.chatResponse.toolCalls: + tool_calls = [ + { + "id": _synthesize_oci_tool_call_id( + i, tc.name, json.dumps(tc.parameters, sort_keys=True) + ), + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.parameters), + }, + } + for i, tc in enumerate(cohere_response.chatResponse.toolCalls) + ] + + content: Optional[str] = response_text if response_text else None + + # Only include ``tool_calls`` in the message dict when actually present. + # Passing an explicit ``None`` would let downstream consumers that key off + # ``"tool_calls" in message`` (rather than truthiness) incorrectly conclude + # that tool calls were attempted. Matches the generic handler's behaviour, + # which only sets ``message.tool_calls`` when tool calls are present. + message: Dict[str, Any] = {"role": "assistant", "content": content} + if tool_calls is not None: + message["tool_calls"] = tool_calls + + model_response.choices = [ + Choices( + index=0, + message=message, + finish_reason=finish_reason, + ) + ] + + usage_info = cohere_response.chatResponse.usage + if usage_info is not None: + model_response.usage = Usage( # type: ignore[attr-defined] + prompt_tokens=usage_info.promptTokens, + completion_tokens=usage_info.completionTokens, + total_tokens=usage_info.totalTokens, + ) + else: + model_response.usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) # type: ignore[attr-defined] + + return model_response + + +def handle_cohere_stream_chunk( + dict_chunk: dict, + prior_tool_calls_emitted: bool = False, + prior_text_emitted: bool = False, +) -> ModelResponseStream: + """Parse a single Cohere SSE chunk into a LiteLLM ModelResponseStream. + + ``prior_tool_calls_emitted`` lets the caller signal whether tool calls + were already emitted in earlier chunks of the same stream. When set, the + terminal consolidation chunk's tool calls are suppressed (they would + duplicate prior deltas); otherwise they are passed through so a stream + that delivers tool calls only on the terminal chunk doesn't silently + drop them. + + ``prior_text_emitted`` plays the analogous role for the ``text`` field: + when set, the terminal consolidation chunk's ``text`` is suppressed + (it would re-emit the full assembled response on top of prior deltas); + when unset (e.g. a degenerate stream that delivers the entire response + in a single SSE event carrying both ``chatHistory`` and ``finishReason``), + the text is passed through so the response content isn't silently lost. + """ + try: + typed_chunk = CohereStreamChunk(**dict_chunk) + except (TypeError, ValidationError) as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as CohereStreamChunk: {str(e)}", + ) + + if typed_chunk.index is None: + typed_chunk.index = 0 + + # OCI Cohere's terminal SSE event re-sends the full assembled response in + # `text` alongside a populated `chatHistory` and a non-null `finishReason`. + # Emitting that text would concatenate the whole response onto the + # already-streamed deltas. We require both signals to be present so that a + # future API change which adds `chatHistory` to intermediate chunks (or a + # rare early-populated case) doesn't silently drop legitimate token deltas. + is_terminal_consolidation = ( + typed_chunk.chatHistory is not None and typed_chunk.finishReason is not None + ) + # On non-terminal text-free chunks (e.g. tool-call-only or keep-alive + # chunks) emit ``content=None`` rather than ``content=""`` so downstream + # stream-mergers that distinguish "no text in this delta" from "an + # explicitly empty text delta" behave correctly. + # + # We only suppress the terminal chunk's ``text`` when the caller has + # confirmed that text deltas were already emitted earlier — otherwise + # (e.g. a degenerate stream that delivers the whole response in a + # single SSE event), passing it through is the only chance to surface it. + text: Optional[str] = ( + None if (is_terminal_consolidation and prior_text_emitted) else typed_chunk.text + ) + + # Tool calls on the terminal consolidation chunk (whether from + # `typed_chunk.toolCalls` or from `chatHistory`) typically restate what + # was already streamed in intermediate chunks. Re-emitting them would + # mint fresh `uuid4` IDs and cause downstream consumers to execute each + # tool call twice. We only suppress when the caller has confirmed that + # tool calls were already emitted earlier — otherwise (e.g. a short + # response that delivers tool calls exclusively on the terminal chunk), + # passing them through is the only chance to surface them. + cohere_tool_calls = ( + None + if (is_terminal_consolidation and prior_tool_calls_emitted) + else typed_chunk.toolCalls + ) + + tool_calls: Optional[List[Dict[str, Any]]] = None + if cohere_tool_calls: + tool_calls = [ + { + # Cohere protocol has no tool-call id, so we synthesize one + # deterministically from the call's content/position. A random + # uuid4 per chunk would cause downstream stream-mergers to + # treat each chunk as a distinct tool call. + "id": _synthesize_oci_tool_call_id( + i, tc.name, json.dumps(tc.parameters, sort_keys=True) + ), + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.parameters), + }, + } + for i, tc in enumerate(cohere_tool_calls) + ] + + finish_reason = _normalize_oci_finish_reason(typed_chunk.finishReason) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index, + delta=Delta( + content=text, + tool_calls=tool_calls, + provider_specific_fields=None, + thinking_blocks=None, + reasoning_content=None, + ), + finish_reason=finish_reason, + ) + ] + ) diff --git a/litellm/llms/oci/chat/generic.py b/litellm/llms/oci/chat/generic.py new file mode 100644 index 0000000000..2cc1ac77a4 --- /dev/null +++ b/litellm/llms/oci/chat/generic.py @@ -0,0 +1,477 @@ +""" +OCI Generative AI — Generic-format chat transformation helpers. + +Handles message building, tool definition adaptation, non-streaming response +parsing, and streaming chunk parsing for models served with +``apiFormat="GENERIC"`` (e.g. Meta Llama, xAI Grok, Google Gemini). +""" + +import datetime +import hashlib +from typing import Any, Dict, List, Optional, Union + +import httpx +from pydantic import ValidationError + +from litellm.llms.oci.common_utils import ( + OCIError, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, +) +from litellm.types.llms.oci import ( + OCICompletionResponse, + OCIContentPartUnion, + OCIImageContentPart, + OCIImageUrl, + OCIMessage, + OCIRoles, + OCIStreamChunk, + OCITextContentPart, + OCIToolCall, + OCIToolDefinition, + OCIVendors, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ( + Delta, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) +from litellm.types.utils import ChatCompletionMessageToolCall, Usage + +# Maps OpenAI role names to OCI GENERIC role names. +open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { + "system": "SYSTEM", + "user": "USER", + "assistant": "ASSISTANT", + "tool": "TOOL", +} + + +# --------------------------------------------------------------------------- +# Message building +# --------------------------------------------------------------------------- + + +def adapt_messages_to_generic_oci_standard_content_message( + role: str, content: Union[str, list] +) -> OCIMessage: + """Convert a plain-text or multipart content message to OCI format.""" + new_content: List[OCIContentPartUnion] = [] + if isinstance(content, str): + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=None, + ) + + for content_item in content: + if not isinstance(content_item, dict): + raise OCIError( + status_code=400, message="Each content item must be a dictionary" + ) + + item_type = content_item.get("type") + if not isinstance(item_type, str): + raise OCIError( + status_code=400, + message="Each content item must have a string `type` field", + ) + if item_type not in ["text", "image_url"]: + raise OCIError( + status_code=400, + message=f"Content type `{item_type}` is not supported by OCI", + ) + + if item_type == "text": + text = content_item.get("text") + if not isinstance(text, str): + raise OCIError( + status_code=400, + message="Content item of type `text` must have a string `text` field", + ) + new_content.append(OCITextContentPart(text=text)) + + elif item_type == "image_url": + image_url = content_item.get("image_url") + if isinstance(image_url, dict): + image_url = image_url.get("url") + if not isinstance(image_url, str): + raise OCIError( + status_code=400, + message="Prop `image_url` must be a string or an object with a `url` property", + ) + new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url))) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=new_content, + toolCalls=None, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_call( + role: str, tool_calls: list +) -> OCIMessage: + """Convert an assistant tool-call message to OCI format.""" + tool_calls_formatted = [] + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + raise OCIError( + status_code=400, message="Each tool call must be a dictionary" + ) + if tool_call.get("type") != "function": + raise OCIError( + status_code=400, message="OCI only supports function tool calls" + ) + + tool_call_id = tool_call.get("id") + if not isinstance(tool_call_id, str): + raise OCIError(status_code=400, message="Tool call `id` must be a string") + + tool_function = tool_call.get("function") + if not isinstance(tool_function, dict): + raise OCIError( + status_code=400, message="Tool call `function` must be a dictionary" + ) + + function_name = tool_function.get("name") + if not isinstance(function_name, str): + raise OCIError( + status_code=400, message="Tool call `function.name` must be a string" + ) + + arguments = tool_call["function"].get("arguments", "{}") + if not isinstance(arguments, str): + raise OCIError( + status_code=400, + message="Tool call `function.arguments` must be a JSON string", + ) + + tool_calls_formatted.append( + OCIToolCall( + id=tool_call_id, + type="FUNCTION", + name=function_name, + arguments=arguments, + ) + ) + + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=None, + toolCalls=tool_calls_formatted, + toolCallId=None, + ) + + +def adapt_messages_to_generic_oci_standard_tool_response( + role: str, tool_call_id: str, content: str +) -> OCIMessage: + """Convert a tool-result message to OCI format.""" + return OCIMessage( + role=open_ai_to_generic_oci_role_map[role], + content=[OCITextContentPart(text=content)], + toolCalls=None, + toolCallId=tool_call_id, + ) + + +def adapt_messages_to_generic_oci_standard( + messages: List[AllMessageValues], +) -> List[OCIMessage]: + """Convert an OpenAI-format message array to OCI GENERIC format.""" + new_messages = [] + for message in messages: + role = message["role"] + content = message.get("content") + tool_calls = message.get("tool_calls") + tool_call_id = message.get("tool_call_id") + + if role == "assistant" and tool_calls is not None: + if not isinstance(tool_calls, list): + raise OCIError( + status_code=400, message="Message `tool_calls` must be a list" + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) + ) + + elif role in ["system", "user", "assistant"] and content is not None: + if not isinstance(content, (str, list)): + raise OCIError( + status_code=400, + message="Message `content` must be a string or list of content parts", + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_content_message(role, content) + ) + + elif role == "tool": + if not isinstance(tool_call_id, str): + raise OCIError( + status_code=400, + message="Tool result message must have a string `tool_call_id`", + ) + if not isinstance(content, str): + raise OCIError( + status_code=400, + message="Tool result message `content` must be a string", + ) + new_messages.append( + adapt_messages_to_generic_oci_standard_tool_response( + role, tool_call_id, content + ) + ) + + return new_messages + + +# --------------------------------------------------------------------------- +# Tool definition adaptation +# --------------------------------------------------------------------------- + + +def adapt_tool_definition_to_oci_standard( + tools: List[Dict], vendor: OCIVendors +) -> List[OCIToolDefinition]: + """Convert OpenAI-format tool definitions to OCI GENERIC format. + + Resolves ``$ref``/``$defs`` and ``anyOf`` that the OCI endpoint rejects. + """ + new_tools = [] + for tool in tools: + if tool["type"] != "function": + raise OCIError(status_code=400, message="OCI only supports function tools") + + tool_function = tool.get("function") + if not isinstance(tool_function, dict): + raise OCIError( + status_code=400, message="Tool `function` must be a dictionary" + ) + + raw_params = tool_function.get("parameters", {}) + resolved_params = sanitize_oci_schema( + resolve_oci_schema_anyof(resolve_oci_schema_refs(raw_params)) + ) + + new_tools.append( + OCIToolDefinition( + type="FUNCTION", + name=tool_function.get("name"), + description=tool_function.get("description", ""), + parameters=resolved_params, + ) + ) + + return new_tools + + +def _normalize_oci_finish_reason(raw: Optional[str]) -> Optional[str]: + """Map an OCI-specific finish reason to its OpenAI-standard equivalent. + + OCI emits ``COMPLETE`` / ``MAX_TOKENS`` / ``TOOL_CALL(S)`` plus a long tail + of error/cancel reasons (``ERROR``, ``ERROR_TOXIC``, ``ERROR_LIMIT``, + ``USER_CANCEL``, ``CONTENT_FILTERED``, ``CANCELLED``, ...). The OpenAI + spec only defines ``stop`` / ``length`` / ``tool_calls`` / ... — anything + else is collapsed to ``"stop"`` so downstream consumers switching on + ``finish_reason`` keep working. A ``None`` input passes through unchanged. + """ + if raw is None: + return None + if raw == "COMPLETE": + return "stop" + if raw == "MAX_TOKENS": + return "length" + if raw in ("TOOL_CALL", "TOOL_CALLS"): + return "tool_calls" + return "stop" + + +def _synthesize_oci_tool_call_id(position: int, name: str, arguments: str) -> str: + """Deterministic synthetic tool-call id derived from chunk content. + + Used as a fallback when OCI omits ``id`` (always the case for the OCI + Cohere protocol, occasionally the case for OCI GENERIC streaming chunks). + A random ``uuid4`` per chunk would cause downstream stream-merging + consumers — which key off the tool-call ``id`` — to treat re-emissions of + the same logical call (e.g. terminal consolidation chunks, retries) as + distinct calls. A content-derived digest stays stable across identical + re-emissions while differing across truly distinct calls. + """ + digest = hashlib.sha256( + f"{position}|{name}|{arguments}".encode("utf-8"), + usedforsecurity=False, + ).hexdigest()[:24] + return f"call_{digest}" + + +def adapt_tools_to_openai_standard( + tools: List[OCIToolCall], +) -> List[ChatCompletionMessageToolCall]: + """Convert OCI tool-call objects in a response to the OpenAI format.""" + return [ + ChatCompletionMessageToolCall( + id=tool.id or _synthesize_oci_tool_call_id(i, tool.name, tool.arguments), + type="function", + function={"name": tool.name, "arguments": tool.arguments}, + ) + for i, tool in enumerate(tools) + ] + + +# --------------------------------------------------------------------------- +# Response parsing +# --------------------------------------------------------------------------- + + +def handle_generic_response( + json_data: dict, + model: str, + model_response: ModelResponse, + raw_response: httpx.Response, +) -> ModelResponse: + """Parse a non-streaming GENERIC OCI response into a LiteLLM ModelResponse.""" + try: + completion_response = OCICompletionResponse(**json_data) + except (TypeError, ValidationError) as e: + raise OCIError( + message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", + status_code=raw_response.status_code, + ) + + iso_str = completion_response.chatResponse.timeCreated + dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) + model_response.created = int(dt.timestamp()) + model_response.model = completion_response.modelId + + if not completion_response.chatResponse.choices: + raise OCIError( + message="OCI response contained no choices", + status_code=raw_response.status_code, + ) + + response_choice = completion_response.chatResponse.choices[0] + message = model_response.choices[0].message # type: ignore + response_message = response_choice.message + if response_message is not None: + if response_message.content: + # Concatenate all text parts — matches the streaming handler, which + # iterates the full content array. Skips non-text parts (e.g. image + # parts) so a leading non-text part doesn't suppress trailing text. + text: Optional[str] = None + for item in response_message.content: + if isinstance(item, OCITextContentPart): + text = (text or "") + item.text + if text is not None: + message.content = text + if response_message.toolCalls: + message.tool_calls = adapt_tools_to_openai_standard( + response_message.toolCalls + ) + + model_response.choices[0].finish_reason = _normalize_oci_finish_reason( # type: ignore[union-attr,assignment] + response_choice.finishReason + ) + + oci_usage = completion_response.chatResponse.usage + reasoning_tokens: Optional[int] = None + if ( + oci_usage.completionTokensDetails + and oci_usage.completionTokensDetails.reasoningTokens is not None + ): + reasoning_tokens = oci_usage.completionTokensDetails.reasoningTokens + model_response.usage = Usage( # type: ignore[attr-defined] + prompt_tokens=oci_usage.promptTokens, + completion_tokens=oci_usage.completionTokens or 0, + total_tokens=oci_usage.totalTokens, + reasoning_tokens=reasoning_tokens, + ) + + return model_response + + +def handle_generic_stream_chunk(dict_chunk: dict) -> ModelResponseStream: + """Parse a single GENERIC SSE chunk into a LiteLLM ModelResponseStream.""" + # OCI streams tool calls progressively — early chunks may omit required fields. + if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"): + for tool_call in dict_chunk["message"]["toolCalls"]: + tool_call.setdefault("arguments", "") + tool_call.setdefault("id", "") + tool_call.setdefault("name", "") + + try: + typed_chunk = OCIStreamChunk(**dict_chunk) + except (TypeError, ValidationError) as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as OCIStreamChunk: {str(e)}", + ) + + if typed_chunk.index is None: + typed_chunk.index = 0 + + # Emit ``content=None`` rather than ``content=""`` on chunks with no text + # parts (e.g. tool-call-only or keep-alive chunks) so downstream + # stream-mergers that distinguish "no text in this delta" from "an + # explicitly empty text delta" behave correctly. + text: Optional[str] = None + if typed_chunk.message and typed_chunk.message.content: + for item in typed_chunk.message.content: + if isinstance(item, OCITextContentPart): + text = (text or "") + item.text + elif isinstance(item, OCIImageContentPart): + raise OCIError( + status_code=500, + message="OCI returned image content in a streaming response — not supported", + ) + else: + raise OCIError( + status_code=500, + message=f"Unsupported content type in OCI streaming response: {item.type}", + ) + + # Build plain tool-call dicts inline (matching the shape produced by + # ``handle_cohere_stream_chunk``) rather than calling + # ``adapt_tools_to_openai_standard`` and ``model_dump``-ing the typed + # objects. Both code paths feed ``Delta.tool_calls``, so emitting the + # same minimal ``{"id", "type", "function": {"name", "arguments"}}`` + # shape keeps downstream stream-mergers behaving identically across + # GENERIC and Cohere chunks. + tool_calls: Optional[List[Dict[str, Any]]] = None + if typed_chunk.message and typed_chunk.message.toolCalls: + tool_calls = [ + { + "id": tc.id or _synthesize_oci_tool_call_id(i, tc.name, tc.arguments), + "type": "function", + "function": { + "name": tc.name, + "arguments": tc.arguments, + }, + } + for i, tc in enumerate(typed_chunk.message.toolCalls) + ] + + finish_reason: Optional[str] = _normalize_oci_finish_reason( + typed_chunk.finishReason + ) + + return ModelResponseStream( + choices=[ + StreamingChoices( + index=typed_chunk.index, + delta=Delta( + content=text, + tool_calls=tool_calls, + provider_specific_fields=None, + thinking_blocks=None, + reasoning_content=None, + ), + finish_reason=finish_reason, + ) + ] + ) diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 62104e921a..f050f9eea3 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -1,20 +1,26 @@ -import base64 -import datetime -import hashlib +""" +OCI Generative AI — chat transformation orchestrator. + +This module wires together the Cohere-specific and Generic-model helpers to +implement the LiteLLM BaseConfig interface. Heavy-lifting lives in: + + - :mod:`litellm.llms.oci.chat.cohere` — Cohere message/tool/response logic + - :mod:`litellm.llms.oci.chat.generic` — Generic message/tool/response logic + - :mod:`litellm.llms.oci.common_utils` — auth, signing, schema utilities +""" + import json -from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, AsyncIterator, Dict, + Iterator, List, Optional, - Protocol, Tuple, Union, ) -from urllib.parse import urlparse import httpx @@ -28,43 +34,43 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, version, ) -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.chat.cohere import ( + _extract_text_content, + adapt_messages_to_cohere_standard, + adapt_tool_definitions_to_cohere_standard, + handle_cohere_response, + handle_cohere_stream_chunk, +) +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_tool_definition_to_oci_standard, + handle_generic_response, + handle_generic_stream_chunk, +) +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + OCIRequestWrapper, # re-exported for backwards compatibility + get_oci_base_url, + resolve_oci_credentials, + sign_oci_request, + validate_oci_environment, +) from litellm.types.llms.oci import ( CohereChatRequest, - CohereMessage, - CohereChatResult, - CohereParameterDefinition, - CohereStreamChunk, - CohereTool, - CohereToolCall, OCIChatRequestPayload, OCICompletionPayload, - OCICompletionResponse, - OCIContentPartUnion, - OCIImageContentPart, - OCIImageUrl, - OCIMessage, - OCIRoles, OCIServingMode, - OCIStreamChunk, - OCITextContentPart, - OCIToolCall, - OCIToolDefinition, OCIVendors, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( - Delta, LlmProviders, ModelResponse, ModelResponseStream, - StreamingChoices, -) -from litellm.utils import ( - ChatCompletionMessageToolCall, - CustomStreamWrapper, - Usage, ) +from litellm.utils import supports_reasoning +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -74,142 +80,157 @@ else: LiteLLMLoggingObj = Any -class OCISignerProtocol(Protocol): - """ - Protocol for OCI request signers (e.g., oci.signer.Signer). - - This protocol defines the interface expected for OCI SDK signer objects. - Compatible with the OCI Python SDK's Signer class. - - See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html - """ - - def do_request_sign( - self, request: Any, *, enforce_content_headers: bool = False - ) -> None: - """ - Sign an HTTP request by adding authentication headers. - - Args: - request: Request object with method, url, headers, body, and path_url attributes - enforce_content_headers: Whether to enforce content-type and content-length headers - """ - ... - - -@dataclass -class OCIRequestWrapper: - """ - Wrapper for HTTP requests compatible with OCI signer interface. - - This class wraps request data in a format compatible with OCI SDK signers, - which expect objects with method, url, headers, body, and path_url attributes. - """ - - method: str - url: str - headers: dict - body: bytes - - @property - def path_url(self) -> str: - """Returns the path + query string for OCI signing.""" - parsed_url = urlparse(self.url) - return parsed_url.path + ("?" + parsed_url.query if parsed_url.query else "") - - -def sha256_base64(data: bytes) -> str: - digest = hashlib.sha256(data).digest() - return base64.b64encode(digest).decode() - - -def build_signature_string(method, path, headers, signed_headers): - lines = [] - for header in signed_headers: - if header == "(request-target)": - value = f"{method.lower()} {path}" - else: - value = headers[header] - lines.append(f"{header}: {value}") - return "\n".join(lines) - - -def load_private_key_from_str(key_str: str): - try: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - except ImportError as e: - raise ImportError( - "cryptography package is required for OCI authentication. " - "Please install it with: pip install cryptography" - ) from e - - key = serialization.load_pem_private_key( - key_str.encode("utf-8"), - password=None, - ) - if not isinstance(key, rsa.RSAPrivateKey): - raise TypeError( - "The provided private key is not an RSA key, which is required for OCI signing." - ) - return key - - -def load_private_key_from_file(file_path: str): - """Loads a private key from a file path""" - try: - with open(file_path, "r", encoding="utf-8") as f: - key_str = f.read().strip() - except FileNotFoundError: - raise FileNotFoundError(f"Private key file not found: {file_path}") - except OSError as e: - raise OSError(f"Failed to read private key file '{file_path}': {e}") from e - - if not key_str: - raise ValueError(f"Private key file is empty: {file_path}") - - return load_private_key_from_str(key_str) - - -def get_vendor_from_model(model: str) -> OCIVendors: - """ - Extracts the vendor from the model name. - - OCI GenAI API uses two apiFormat values: - - "COHERE" for Cohere models (command-r, command-a, etc.) - - "GENERIC" for all other models (Meta Llama, xAI Grok, Google Gemini, etc.) - - Args: - model (str): The model name (e.g., "cohere.command-a-03-2025", "meta.llama-3.3-70b-instruct"). - Returns: - OCIVendors: The vendor enum value. - """ - vendor = model.split(".")[0].lower() - if vendor == "cohere": - return OCIVendors.COHERE - else: - return OCIVendors.GENERIC - - -# 5 minute timeout (models may need to load) +# Streaming timeout — generous because OCI models may need to warm up on first request STREAMING_TIMEOUT = 60 * 5 +def _model_uses_max_completion_tokens(model: str) -> bool: + """Return True for OCI-hosted models that require ``maxCompletionTokens``. + + Reasoning models on OCI (e.g. the OpenAI GPT-5 family) reject ``maxTokens`` + with HTTP 400 and require ``maxCompletionTokens`` per OpenAI's reasoning-API + convention. Driven by ``supports_reasoning`` in + ``model_prices_and_context_window.json`` so new model families are picked + up via a catalog update rather than a code change. + """ + if not model: + return False + name = model[4:] if model.lower().startswith("oci/") else model + return supports_reasoning(model=name, custom_llm_provider="oci") + + +def _iter_sse_events(stream: Iterator[str]) -> Iterator[str]: + """Yield one ``data:`` SSE line at a time from a sync text stream. + + The OCI streaming endpoint does not align SSE event boundaries with HTTP + read boundaries. A single read may carry multiple events, a single event + may straddle two reads, and some events arrive separated by only ``\\n`` + instead of ``\\n\\n``. This helper buffers across reads and yields each + complete ``data:`` line so JSON parsing downstream never sees a partial + payload. + """ + buffer = "" + for item in stream: + buffer += item + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + stripped = line.strip() + if stripped.startswith("data:"): + yield stripped + stripped = buffer.strip() + if stripped.startswith("data:"): + yield stripped + + +async def _aiter_sse_events(stream: AsyncIterator[str]) -> AsyncIterator[str]: + """Async twin of :func:`_iter_sse_events`.""" + buffer = "" + async for item in stream: + buffer += item + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + stripped = line.strip() + if stripped.startswith("data:"): + yield stripped + stripped = buffer.strip() + if stripped.startswith("data:"): + yield stripped + + +def _normalize_tool_choice(selected_params: Dict) -> None: + tc = selected_params.get("toolChoice") + if tc is None: + return + if isinstance(tc, str): + tc_map = { + "auto": {"type": "AUTO"}, + "none": {"type": "NONE"}, + "required": {"type": "REQUIRED"}, + "any": {"type": "REQUIRED"}, + } + selected_params["toolChoice"] = tc_map.get( + tc.lower(), {"type": "FUNCTION", "name": tc} + ) + return + if isinstance(tc, dict): + raw_type = tc.get("type") + if not isinstance(raw_type, str): + raise OCIError( + status_code=400, + message=f"Invalid tool_choice for OCI: missing or non-string 'type' in {tc!r}", + ) + upper = raw_type.upper() + if upper == "FUNCTION": + fn = tc.get("function") + name = fn.get("name") if isinstance(fn, dict) else tc.get("name") + if not (isinstance(name, str) and name): + raise OCIError( + status_code=400, + message="Invalid tool_choice for OCI: 'FUNCTION' type requires a non-empty function name", + ) + selected_params["toolChoice"] = {"type": "FUNCTION", "name": name} + elif upper in {"AUTO", "NONE", "REQUIRED"}: + selected_params["toolChoice"] = {"type": upper} + else: + raise OCIError( + status_code=400, + message=( + f"Invalid tool_choice for OCI: unsupported type {raw_type!r}; " + "expected one of 'FUNCTION', 'AUTO', 'NONE', 'REQUIRED'" + ), + ) + return + raise OCIError( + status_code=400, + message=( + f"Invalid tool_choice for OCI: expected str or dict, got " + f"{type(tc).__name__}" + ), + ) + + +def _normalize_response_format(selected_params: Dict, vendor: OCIVendors) -> None: + rf = selected_params.get("responseFormat") + if not isinstance(rf, dict) or "type" not in rf: + return + rf_payload = dict(rf) + selected_params["responseFormat"] = rf_payload + response_type = rf_payload["type"] + if "json_schema" in rf_payload: + raw_schema = rf_payload.pop("json_schema") + rf_payload["jsonSchema"] = ( + dict(raw_schema) if isinstance(raw_schema, dict) else raw_schema + ) + if vendor == OCIVendors.COHERE: + rf_payload["type"] = response_type + else: + fmt = response_type.upper() + rf_payload["type"] = "JSON_OBJECT" if fmt == "JSON" else fmt + + +def get_vendor_from_model(model: str) -> OCIVendors: + """Return the OCI vendor enum for a model name. + + OCI GenAI uses two ``apiFormat`` values: + + - ``"COHERE"`` for Cohere models (``cohere.*``) + - ``"GENERIC"`` for all others (Meta Llama, xAI Grok, Google Gemini, …) + """ + name = model[4:] if model.lower().startswith("oci/") else model + vendor = name.split(".")[0].lower() + if vendor == "cohere": + return OCIVendors.COHERE + return OCIVendors.GENERIC + + class OCIChatConfig(BaseConfig): - """ - Configuration class for OCI's API interface. - """ + """LiteLLM BaseConfig implementation for OCI Generative AI chat.""" - def __init__( - self, - ) -> None: - locals_ = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) - # mark the class as using a custom stream wrapper because the default only iterates on lines - setattr(self.__class__, "has_custom_stream_wrapper", True) + @property + def has_custom_stream_wrapper(self) -> bool: + return True + def __init__(self) -> None: self.openai_to_oci_generic_param_map = { "stream": "isStream", "max_tokens": "maxTokens", @@ -221,6 +242,7 @@ class OCIChatConfig(BaseConfig): "logit_bias": "logitBias", "n": "numGenerations", "presence_penalty": "presencePenalty", + "reasoning_effort": "reasoningEffort", "seed": "seed", "stop": "stop", "tool_choice": "toolChoice", @@ -239,25 +261,43 @@ class OCIChatConfig(BaseConfig): "response_format": "responseFormat", } - # Cohere and Gemini use the same parameter mapping as GENERIC - self.openai_to_oci_cohere_param_map = ( - self.openai_to_oci_generic_param_map.copy() - ) + # Cohere param map differs from GENERIC in three ways: + # - tool_choice is unsupported + # - stop sequences key is "stopSequences" not "stop" + # - n (numGenerations) is GENERIC-only + # The unsupported keys are kept in the map with value ``False`` so + # ``map_openai_params`` either drops them (under drop_params) or raises + # a clear error, rather than silently passing them through. + self.openai_to_oci_cohere_param_map = { + k: ("stopSequences" if k == "stop" else v) + for k, v in self.openai_to_oci_generic_param_map.items() + } + self.openai_to_oci_cohere_param_map["tool_choice"] = False + self.openai_to_oci_cohere_param_map["n"] = False + # ``top_k`` is not a standard OpenAI param, but Cohere's chat request + # accepts ``topK`` and LiteLLM commonly forwards ``top_k`` as a + # passthrough param. Cohere-only — ``OCIChatRequestPayload`` (GENERIC) + # has no ``topK`` field. + self.openai_to_oci_cohere_param_map["top_k"] = "topK" + # OCI Cohere models are not reasoning models; mark reasoning_effort + # explicitly unsupported so callers either get a clear error or have + # the param dropped under drop_params, rather than silently passing + # through and tripping Pydantic validation on CohereChatRequest. + self.openai_to_oci_cohere_param_map["reasoning_effort"] = False + # CohereChatRequest has no logProbs/logitBias fields, so passing these + # through would be silently dropped by Pydantic. Mark them unsupported + # so get_supported_openai_params doesn't advertise them and callers + # get a clear error (or drop_params behaviour) instead. + self.openai_to_oci_cohere_param_map["logprobs"] = False + self.openai_to_oci_cohere_param_map["logit_bias"] = False def get_supported_openai_params(self, model: str) -> List[str]: - supported_params = [] - vendor = get_vendor_from_model(model) - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - open_ai_to_oci_param_map.pop("tool_choice") - open_ai_to_oci_param_map.pop("max_retries") - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map - for key, value in open_ai_to_oci_param_map.items(): - if value: - supported_params.append(key) - - return supported_params + param_map = ( + self.openai_to_oci_cohere_param_map + if get_vendor_from_model(model) == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + return [key for key, value in param_map.items() if value] def map_openai_params( self, @@ -268,238 +308,34 @@ class OCIChatConfig(BaseConfig): ) -> dict: adapted_params = {} vendor = get_vendor_from_model(model) - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map - - all_params = {**non_default_params, **optional_params} - - for key, value in all_params.items(): - alias = open_ai_to_oci_param_map.get(key) + param_map = ( + self.openai_to_oci_cohere_param_map + if vendor == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + for key, value in {**non_default_params, **optional_params}.items(): + alias = param_map.get(key) if alias is False: - # Workaround for mypy issue if drop_params or litellm.drop_params: continue - raise Exception(f"param `{key}` is not supported on OCI") - + raise OCIError( + status_code=400, + message=f"param `{key}` is not supported on OCI", + ) if alias is None: adapted_params[key] = value continue - adapted_params[alias] = value - + # Preserve the original OpenAI ``response_format`` key alongside the + # OCI-mapped ``responseFormat`` so downstream litellm framework code + # (e.g. ``json_mode`` detection, logging) that inspects + # ``optional_params["response_format"]`` continues to work. if alias == "responseFormat": adapted_params["response_format"] = value return adapted_params - def _sign_with_oci_signer( - self, - headers: dict, - optional_params: dict, - request_data: dict, - api_base: str, - ) -> Tuple[dict, bytes]: - """ - Sign request using OCI SDK Signer object. - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including oci_signer - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - - Returns: - Tuple of (signed_headers, encoded_body) - - Raises: - OCIError: If signing fails - ValueError: If HTTP method is unsupported - """ - oci_signer = optional_params.get("oci_signer") - body = json.dumps(request_data).encode("utf-8") - method = str(optional_params.get("method", "POST")).upper() - - if method not in ["POST", "GET", "PUT", "DELETE", "PATCH"]: - raise ValueError(f"Unsupported HTTP method: {method}") - - prepared_headers = headers.copy() - prepared_headers.setdefault("content-type", "application/json") - prepared_headers.setdefault("content-length", str(len(body))) - - request_wrapper = OCIRequestWrapper( - method=method, url=api_base, headers=prepared_headers, body=body - ) - - if oci_signer is None: - raise ValueError( - "oci_signer cannot be None when calling _sign_with_oci_signer" - ) - - try: - oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True) - except Exception as e: - raise OCIError( - status_code=500, - message=( - f"Failed to sign request with provided oci_signer: {str(e)}. " - "The signer must implement the OCI SDK Signer interface with a " - "do_request_sign(request, enforce_content_headers=True) method. " - "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" - ), - ) from e - - headers.update(request_wrapper.headers) - return headers, body - - def _sign_with_manual_credentials( - self, - headers: dict, - optional_params: dict, - request_data: dict, - api_base: str, - ) -> Tuple[dict, None]: - """ - Sign request using manual OCI credentials. - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including OCI credentials - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - - Returns: - Tuple of (signed_headers, None) - - Raises: - Exception: If required credentials are missing - ImportError: If cryptography package is not installed - """ - oci_region = optional_params.get("oci_region", "us-ashburn-1") - api_base = ( - api_base - or litellm.api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " - "and at least one of oci_key or oci_key_file." - ) - - method = str(optional_params.get("method", "POST")).upper() - body = json.dumps(request_data).encode("utf-8") - parsed = urlparse(api_base) - path = parsed.path or "/" - host = parsed.netloc - - date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - content_type = headers.get("content-type", "application/json") - content_length = str(len(body)) - x_content_sha256 = sha256_base64(body) - - headers_to_sign = { - "date": date, - "host": host, - "content-type": content_type, - "content-length": content_length, - "x-content-sha256": x_content_sha256, - } - - signed_headers = [ - "date", - "(request-target)", - "host", - "content-length", - "content-type", - "x-content-sha256", - ] - signing_string = build_signature_string( - method, path, headers_to_sign, signed_headers - ) - - try: - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.asymmetric import padding - except ImportError as e: - raise ImportError( - "cryptography package is required for OCI authentication. " - "Please install it with: pip install cryptography" - ) from e - - # Handle oci_key - it should be a string (PEM content) - oci_key_content = None - if oci_key: - if isinstance(oci_key, str): - oci_key_content = oci_key - # Fix common issues with PEM content - # Replace escaped newlines with actual newlines - oci_key_content = oci_key_content.replace("\\n", "\n") - # Ensure proper line endings - if "\r\n" in oci_key_content: - oci_key_content = oci_key_content.replace("\r\n", "\n") - else: - raise OCIError( - status_code=400, - message=f"oci_key must be a string containing the PEM private key content. " - f"Got type: {type(oci_key).__name__}", - ) - - private_key = ( - load_private_key_from_str(oci_key_content) - if oci_key_content - else load_private_key_from_file(oci_key_file) if oci_key_file else None - ) - - if private_key is None: - raise OCIError( - status_code=400, - message="Private key is required for OCI authentication. Please provide either oci_key or oci_key_file.", - ) - - signature = private_key.sign( - signing_string.encode("utf-8"), - padding.PKCS1v15(), - hashes.SHA256(), - ) - signature_b64 = base64.b64encode(signature).decode() - - key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" - - authorization = ( - 'Signature version="1",' - f'keyId="{key_id}",' - 'algorithm="rsa-sha256",' - f'headers="{" ".join(signed_headers)}",' - f'signature="{signature_b64}"' - ) - - headers.update( - { - "authorization": authorization, - "date": date, - "host": host, - "content-type": content_type, - "content-length": content_length, - "x-content-sha256": x_content_sha256, - } - ) - - return headers, None - def sign_request( self, headers: dict, @@ -510,61 +346,16 @@ class OCIChatConfig(BaseConfig): model: Optional[str] = None, stream: Optional[bool] = None, fake_stream: Optional[bool] = None, - ) -> Tuple[dict, Optional[bytes]]: - """ - Sign the OCI request by adding authentication headers. - - Supports two signing modes: - 1. OCI SDK Signer: Use an oci_signer object to sign the request - 2. Manual Signing: Use OCI credentials to manually sign the request - - Args: - headers: Request headers to be signed - optional_params: Optional parameters including auth credentials or oci_signer - request_data: The request body dict to be sent in HTTP request - api_base: The complete URL for the HTTP request - api_key: Optional API key (not used for OCI) - model: Optional model name - stream: Optional streaming flag - fake_stream: Optional fake streaming flag - - Returns: - Tuple of (signed_headers, encoded_body): - - If oci_signer is provided: Returns (headers, body) where body is the encoded JSON - - If manual credentials are provided: Returns (headers, None) as body is not returned - for the manual signing path - - Raises: - OCIError: If signing fails with oci_signer - Exception: If required credentials are missing - ImportError: If cryptography package is not installed (manual signing only) - - Example: - >>> from oci.signer import Signer - >>> signer = Signer( - ... tenancy="ocid1.tenancy.oc1..", - ... user="ocid1.user.oc1..", - ... fingerprint="xx:xx:xx", - ... private_key_file_location="~/.oci/key.pem" - ... ) - >>> headers, body = config.sign_request( - ... headers={}, - ... optional_params={"oci_signer": signer}, - ... request_data={"message": "Hello"}, - ... api_base="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/..." - ... ) - """ - oci_signer = optional_params.get("oci_signer") - - # If a signer is provided, use it for request signing - if oci_signer is not None: - return self._sign_with_oci_signer( - headers, optional_params, request_data, api_base - ) - - # Standard manual credential signing - return self._sign_with_manual_credentials( - headers, optional_params, request_data, api_base + ) -> Tuple[dict, bytes]: + return sign_oci_request( + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, ) def validate_environment( @@ -577,80 +368,35 @@ class OCIChatConfig(BaseConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - """ - Validate the OCI environment and credentials. - - Supports two authentication modes: - 1. OCI SDK Signer: Pass an oci_signer object (e.g., oci.signer.Signer) - 2. Manual Credentials: Pass oci_user, oci_fingerprint, oci_tenancy, and oci_key/oci_key_file - - Args: - headers: Request headers to populate - model: Model name - messages: List of chat messages - optional_params: Optional parameters including authentication credentials - litellm_params: LiteLLM parameters - api_key: Optional API key (not used for OCI) - api_base: Optional API base URL - - Returns: - Updated headers dict - - Raises: - Exception: If required parameters are missing or invalid - """ - oci_signer = optional_params.get("oci_signer") - oci_region = optional_params.get("oci_region", "us-ashburn-1") - - # Determine api_base - api_base = ( - api_base - or litellm.api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - - if not api_base: - raise Exception( - "Either `api_base` must be provided or `litellm.api_base` must be set. " - "Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." - ) - - # Validate credentials only if signer is not provided - if oci_signer is None: - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - oci_compartment_id = optional_params.get("oci_compartment_id") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - or not oci_compartment_id - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " - "and at least one of oci_key or oci_key_file. " - "Alternatively, provide an oci_signer object from the OCI SDK." - ) - - # Common header setup - headers.update( - { - "content-type": "application/json", - "user-agent": f"litellm/{version}", - } - ) - if not messages: - raise Exception( - "kwarg `messages` must be an array of messages that follow the openai chat standard" + raise OCIError( + status_code=400, + message="kwarg `messages` must be an array of messages that follow the openai chat standard", ) - - return headers + if optional_params.get("oci_signer") is None: + creds = resolve_oci_credentials(optional_params) + missing = [ + k + for k in ( + "oci_user", + "oci_fingerprint", + "oci_tenancy", + "oci_compartment_id", + ) + if not creds.get(k) + ] + if missing or not (creds.get("oci_key") or creds.get("oci_key_file")): + raise OCIError( + status_code=401, + message=( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " + "oci_compartment_id and at least one of oci_key or oci_key_file. " + "These can be supplied via optional_params or via OCI_USER, OCI_FINGERPRINT, " + "OCI_TENANCY, OCI_COMPARTMENT_ID, OCI_KEY_FILE environment variables. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + return validate_oci_environment(headers, optional_params, api_key) def get_complete_url( self, @@ -661,43 +407,63 @@ class OCIChatConfig(BaseConfig): litellm_params: dict, stream: Optional[bool] = None, ) -> str: - oci_region = optional_params.get("oci_region", "us-ashburn-1") - return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat" + base = get_oci_base_url(optional_params, api_base or litellm.api_base) + return f"{base}/{OCI_API_VERSION}/actions/chat" - def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict: - selected_params = {} - if vendor == OCIVendors.COHERE: - open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map - # remove tool_choice from the map - open_ai_to_oci_param_map.pop("tool_choice") - # Add default values for Cohere API - selected_params = { - "maxTokens": 600, - "temperature": 1, - "topK": 0, - "topP": 0.75, - "frequencyPenalty": 0, - } - else: - open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map + def _get_optional_params( + self, vendor: OCIVendors, optional_params: dict, model: str = "" + ) -> Dict: + param_map = ( + self.openai_to_oci_cohere_param_map + if vendor == OCIVendors.COHERE + else self.openai_to_oci_generic_param_map + ) + selected_params: Dict = {} - # Map OpenAI params to OCI params - for openai_key, oci_key in open_ai_to_oci_param_map.items(): - if oci_key and openai_key in optional_params: - selected_params[oci_key] = optional_params[openai_key] # type: ignore[index] + # OpenAI reasoning models on OCI (e.g. GPT-5 family) reject "maxTokens" + # and require "maxCompletionTokens" per OCI's /20231130/Chat schema. + # Driven by the supports_reasoning flag in the model catalog. Cohere's + # endpoint uses "maxTokens" regardless, so the override is GENERIC-only. + max_tokens_key = ( + "maxCompletionTokens" + if vendor != OCIVendors.COHERE + and model + and _model_uses_max_completion_tokens(model) + else "maxTokens" + ) - # Also check for already-mapped OCI params (for backward compatibility) - for oci_value in open_ai_to_oci_param_map.values(): - if ( - oci_value - and oci_value in optional_params - and oci_value not in selected_params - ): - selected_params[oci_value] = optional_params[oci_value] # type: ignore[index] + # ``map_openai_params`` runs before ``transform_request`` (and thus + # before this helper), so by the time we see ``optional_params`` the + # OpenAI keys have already been translated to their OCI aliases. + # We still accept the original OpenAI key as a fallback for callers + # that build ``optional_params`` directly, with OpenAI keys winning + # over OCI aliases when both happen to be present. The first OpenAI + # key reaching a given OCI target wins, so ``max_tokens`` / + # ``max_completion_tokens`` (both → ``maxTokens``) don't double-write. + for openai_key, oci_alias in param_map.items(): + if not oci_alias: + continue + target = max_tokens_key if oci_alias == "maxTokens" else oci_alias + if target in selected_params: + continue + if openai_key in optional_params: + selected_params[target] = optional_params[openai_key] # type: ignore[index] + elif oci_alias in optional_params: + selected_params[target] = optional_params[oci_alias] # type: ignore[index] + + # OCI expects uppercase reasoning levels (LOW/MEDIUM/HIGH/NONE); OpenAI + # clients send lowercase. OpenAI's "disable" maps to OCI's "NONE". + if "reasoningEffort" in selected_params: + effort = selected_params["reasoningEffort"] + if isinstance(effort, str): + normalized = effort.upper() + if normalized == "DISABLE": + normalized = "NONE" + selected_params["reasoningEffort"] = normalized if "tools" in selected_params: if vendor == OCIVendors.COHERE: - selected_params["tools"] = self.adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] + selected_params["tools"] = adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] selected_params["tools"] # type: ignore[arg-type] ) else: @@ -705,146 +471,15 @@ class OCIChatConfig(BaseConfig): selected_params["tools"], vendor # type: ignore[arg-type] ) - # Transform response_format type to OCI uppercase format - if "responseFormat" in selected_params: - rf = selected_params["responseFormat"] - if isinstance(rf, dict) and "type" in rf: - rf_payload = dict(rf) - selected_params["responseFormat"] = rf_payload + # Normalise tool_choice to OCI's flat uppercase dict form + # ({"type": "AUTO"|"NONE"|"REQUIRED"} or {"type": "FUNCTION", "name": ""}). + # OCI rejects both the OpenAI string and the nested OpenAI dict shape. + _normalize_tool_choice(selected_params) - response_type = rf_payload["type"] - schema_payload: Optional[Any] = None - - if "json_schema" in rf_payload: - raw_schema_payload = rf_payload.pop("json_schema") - if isinstance(raw_schema_payload, dict): - schema_payload = dict(raw_schema_payload) - else: - schema_payload = raw_schema_payload - - if schema_payload is not None: - rf_payload["jsonSchema"] = schema_payload - - if vendor == OCIVendors.COHERE: - # Cohere expects lower-case type values - rf_payload["type"] = response_type - else: - format_type = response_type.upper() - if format_type == "JSON": - format_type = "JSON_OBJECT" - rf_payload["type"] = format_type + _normalize_response_format(selected_params, vendor) return selected_params - def adapt_messages_to_cohere_standard( - self, messages: List[AllMessageValues] - ) -> List[CohereMessage]: - """Build chat history for Cohere models.""" - chat_history = [] - for msg in messages[:-1]: # All messages except the last one - role = msg.get("role") - content = msg.get("content") - - if isinstance(content, list): - # Extract text from content array - text_content = "" - for content_item in content: - if ( - isinstance(content_item, dict) - and content_item.get("type") == "text" - ): - text_content += content_item.get("text", "") - content = text_content - - # Ensure content is a string - if not isinstance(content, str): - content = str(content) if content is not None else "" - - # Handle tool calls - tool_calls: Optional[List[CohereToolCall]] = None - if role == "assistant" and "tool_calls" in msg and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] - tool_calls = [] - for tool_call in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item] - # Parse arguments if they're a JSON string - raw_arguments: Any = tool_call.get("function", {}).get( - "arguments", {} - ) - if isinstance(raw_arguments, str): - try: - arguments: Dict[str, Any] = json.loads(raw_arguments) - except json.JSONDecodeError: - arguments = {} - else: - arguments = raw_arguments - - tool_calls.append( - CohereToolCall( - name=str(tool_call.get("function", {}).get("name", "")), - parameters=arguments, - ) - ) - - if role == "user": - chat_history.append(CohereMessage(role="USER", message=content)) - elif role == "assistant": - chat_history.append( - CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls) - ) - elif role == "tool": - # Tool messages need special handling - chat_history.append( - CohereMessage( - role="TOOL", - message=content, - toolCalls=None, # Tool messages don't have tool calls - ) - ) - - return chat_history - - def adapt_tool_definitions_to_cohere_standard( - self, tools: List[Dict[str, Any]] - ) -> List[CohereTool]: - """Adapt tool definitions to Cohere format.""" - cohere_tools = [] - for tool in tools: - function_def = tool.get("function", {}) - parameters = function_def.get("parameters", {}).get("properties", {}) - required = function_def.get("parameters", {}).get("required", []) - - parameter_definitions = {} - for param_name, param_schema in parameters.items(): - parameter_definitions[param_name] = CohereParameterDefinition( - description=param_schema.get("description", ""), - type=param_schema.get("type", "string"), - isRequired=param_name in required, - ) - - cohere_tools.append( - CohereTool( - name=function_def.get("name", ""), - description=function_def.get("description", ""), - parameterDefinitions=parameter_definitions, - ) - ) - - return cohere_tools - - def _extract_text_content(self, content: Any) -> str: - """Extract text content from message content.""" - if isinstance(content, str): - return content - elif isinstance(content, list): - text_content = "" - for content_item in content: - if ( - isinstance(content_item, dict) - and content_item.get("type") == "text" - ): - text_content += content_item.get("text", "") - return text_content - return str(content) - def transform_request( self, model: str, @@ -853,186 +488,78 @@ class OCIChatConfig(BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - oci_compartment_id = optional_params.get("oci_compartment_id", None) + creds = resolve_oci_credentials(optional_params) + oci_compartment_id = creds["oci_compartment_id"] if not oci_compartment_id: - raise Exception("kwarg `oci_compartment_id` is required for OCI requests") + raise OCIError( + status_code=400, + message=( + "oci_compartment_id is required for OCI chat requests. " + "Pass it as optional_params or set the OCI_COMPARTMENT_ID env var." + ), + ) vendor = get_vendor_from_model(model) oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") if oci_serving_mode not in ["ON_DEMAND", "DEDICATED"]: - raise Exception( - "kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'" + raise OCIError( + status_code=400, + message="kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'", ) if oci_serving_mode == "DEDICATED": - oci_endpoint_id = optional_params.get("oci_endpoint_id", model) - servingMode = OCIServingMode( + serving_mode = OCIServingMode( servingType="DEDICATED", - endpointId=oci_endpoint_id, + endpointId=optional_params.get("oci_endpoint_id", model), ) else: - servingMode = OCIServingMode( - servingType="ON_DEMAND", - modelId=model, - ) + serving_mode = OCIServingMode(servingType="ON_DEMAND", modelId=model) - # Build request based on vendor type if vendor == OCIVendors.COHERE: - # For Cohere, we need to use the specific Cohere format - # Extract the last user message as the main message - user_messages = [msg for msg in messages if msg.get("role") == "user"] + user_messages = [m for m in messages if m.get("role") == "user"] if not user_messages: - raise Exception("No user message found for Cohere model") + raise OCIError( + status_code=400, + message="No user message found — Cohere models require at least one user message", + ) - # Extract system messages into preambleOverride - system_messages = [msg for msg in messages if msg.get("role") == "system"] + system_messages = [m for m in messages if m.get("role") == "system"] preamble_override = None if system_messages: preamble = "\n".join( - self._extract_text_content(msg["content"]) - for msg in system_messages + _extract_text_content(m["content"]) for m in system_messages ) if preamble: preamble_override = preamble - # Create Cohere-specific chat request - optional_cohere_params = self._get_optional_params( - OCIVendors.COHERE, optional_params - ) chat_request = CohereChatRequest( apiFormat="COHERE", - message=self._extract_text_content(user_messages[-1]["content"]), - chatHistory=self.adapt_messages_to_cohere_standard(messages), + message=_extract_text_content(user_messages[-1]["content"]), + chatHistory=adapt_messages_to_cohere_standard( + [m for m in messages if m.get("role") != "system"] + ), preambleOverride=preamble_override, - **optional_cohere_params, + **self._get_optional_params(OCIVendors.COHERE, optional_params, model), ) - data = OCICompletionPayload( compartmentId=oci_compartment_id, - servingMode=servingMode, + servingMode=serving_mode, chatRequest=chat_request, ) else: - # Use generic format for other vendors data = OCICompletionPayload( compartmentId=oci_compartment_id, - servingMode=servingMode, + servingMode=serving_mode, chatRequest=OCIChatRequestPayload( apiFormat=vendor.value, messages=adapt_messages_to_generic_oci_standard(messages), - **self._get_optional_params(vendor, optional_params), + **self._get_optional_params(vendor, optional_params, model), ), ) return data.model_dump(exclude_none=True) - def _handle_cohere_response( - self, json_response: dict, model: str, model_response: ModelResponse - ) -> ModelResponse: - """Handle Cohere-specific response format.""" - cohere_response = CohereChatResult(**json_response) - # Cohere response format (uses camelCase) - model_id = model - - # Set basic response info - model_response.model = model_id - model_response.created = int(datetime.datetime.now().timestamp()) - - # Extract the response text - response_text = cohere_response.chatResponse.text - oci_finish_reason = cohere_response.chatResponse.finishReason - - # Map finish reason - if oci_finish_reason == "COMPLETE": - finish_reason = "stop" - elif oci_finish_reason == "MAX_TOKENS": - finish_reason = "length" - else: - finish_reason = "stop" - - # Handle tool calls - tool_calls: Optional[List[Dict[str, Any]]] = None - if cohere_response.chatResponse.toolCalls: - tool_calls = [] - for tool_call in cohere_response.chatResponse.toolCalls: - tool_calls.append( - { - "id": f"call_{len(tool_calls)}", # Generate a simple ID - "type": "function", - "function": { - "name": tool_call.name, - "arguments": json.dumps(tool_call.parameters), - }, - } - ) - - # Create choice - from litellm.types.utils import Choices - - choice = Choices( - index=0, - message={ - "role": "assistant", - "content": response_text, - "tool_calls": tool_calls, - }, - finish_reason=finish_reason, - ) - model_response.choices = [choice] - - # Extract usage info - usage_info = cohere_response.chatResponse.usage - from litellm.types.utils import Usage - - model_response.usage = Usage( # type: ignore[attr-defined] - prompt_tokens=usage_info.promptTokens, # type: ignore[union-attr] - completion_tokens=usage_info.completionTokens, # type: ignore[union-attr] - total_tokens=usage_info.totalTokens, # type: ignore[union-attr] - ) - - return model_response - - def _handle_generic_response( - self, - json: dict, - model: str, - model_response: ModelResponse, - raw_response: httpx.Response, - ) -> ModelResponse: - """Handle generic OCI response format.""" - try: - completion_response = OCICompletionResponse(**json) - except TypeError as e: - raise OCIError( - message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", - status_code=raw_response.status_code, - ) - - iso_str = completion_response.chatResponse.timeCreated - dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) - model_response.created = int(dt.timestamp()) - - model_response.model = completion_response.modelId - - message = model_response.choices[0].message # type: ignore - response_message = completion_response.chatResponse.choices[0].message - if response_message.content and response_message.content[0].type == "TEXT": - message.content = response_message.content[0].text - if response_message.toolCalls: - message.tool_calls = adapt_tools_to_openai_standard( - response_message.toolCalls - ) - - usage = Usage( - prompt_tokens=completion_response.chatResponse.usage.promptTokens, - completion_tokens=completion_response.chatResponse.usage.completionTokens, - total_tokens=completion_response.chatResponse.usage.totalTokens, - ) - model_response.usage = usage # type: ignore - - return model_response - def transform_response( self, model: str, @@ -1047,34 +574,31 @@ class OCIChatConfig(BaseConfig): api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: - json = raw_response.json() # noqa: F811 + response_json = raw_response.json() - error = json.get("error") - - if error is not None: - raise OCIError( - message=str(json["error"]), - status_code=raw_response.status_code, - ) - - if not isinstance(json, dict): + if not isinstance(response_json, dict): raise OCIError( message="Invalid response format from OCI", status_code=raw_response.status_code, ) - vendor = get_vendor_from_model(model) + if response_json.get("error") is not None: + raise OCIError( + message=str(response_json["error"]), + status_code=raw_response.status_code, + ) - # Handle response based on vendor type + vendor = get_vendor_from_model(model) if vendor == OCIVendors.COHERE: - model_response = self._handle_cohere_response(json, model, model_response) + model_response = handle_cohere_response( + response_json, model, model_response, raw_response + ) else: - model_response = self._handle_generic_response( - json, model, model_response, raw_response + model_response = handle_generic_response( + response_json, model, model_response, raw_response ) model_response._hidden_params["additional_headers"] = raw_response.headers - return model_response @track_llm_api_timing() @@ -1091,8 +615,6 @@ class OCIChatConfig(BaseConfig): json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, ) -> "OCIStreamWrapper": - if "stream" in data: - del data["stream"] if client is None or isinstance(client, AsyncHTTPHandler): client = _get_httpx_client(params={}) @@ -1100,7 +622,11 @@ class OCIChatConfig(BaseConfig): response = client.post( api_base, headers=headers, - data=json.dumps(data), + data=( + signed_json_body + if signed_json_body is not None + else json.dumps(data) + ), stream=True, logging_obj=logging_obj, timeout=STREAMING_TIMEOUT, @@ -1111,15 +637,12 @@ class OCIChatConfig(BaseConfig): if response.status_code != 200: raise OCIError(status_code=response.status_code, message=response.text) - completion_stream = response.iter_text() - - streaming_response = OCIStreamWrapper( - completion_stream=completion_stream, + return OCIStreamWrapper( + completion_stream=_iter_sse_events(response.iter_text()), model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - return streaming_response @track_llm_api_timing() async def get_async_custom_stream_wrapper( @@ -1135,17 +658,18 @@ class OCIChatConfig(BaseConfig): json_mode: Optional[bool] = None, signed_json_body: Optional[bytes] = None, ) -> "OCIStreamWrapper": - if "stream" in data: - del data["stream"] - if client is None or isinstance(client, HTTPHandler): - client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={}) + client = get_async_httpx_client(llm_provider=LlmProviders.OCI, params={}) try: response = await client.post( api_base, headers=headers, - data=json.dumps(data), + data=( + signed_json_body + if signed_json_body is not None + else json.dumps(data) + ), stream=True, logging_obj=logging_obj, timeout=STREAMING_TIMEOUT, @@ -1156,22 +680,12 @@ class OCIChatConfig(BaseConfig): if response.status_code != 200: raise OCIError(status_code=response.status_code, message=response.text) - completion_stream = response.aiter_text() - - async def split_chunks(completion_stream: AsyncIterator[str]): - async for item in completion_stream: - for chunk in item.split("\n\n"): - if not chunk: - continue - yield chunk.strip() - - streaming_response = OCIStreamWrapper( - completion_stream=split_chunks(completion_stream), + return OCIStreamWrapper( + completion_stream=_aiter_sse_events(response.aiter_text()), model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - return streaming_response def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] @@ -1179,332 +693,61 @@ class OCIChatConfig(BaseConfig): return OCIError(status_code=status_code, message=error_message) -open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = { - "system": "SYSTEM", - "user": "USER", - "assistant": "ASSISTANT", - "tool": "TOOL", -} - - -def adapt_messages_to_generic_oci_standard_content_message( - role: str, content: Union[str, list] -) -> OCIMessage: - new_content: List[OCIContentPartUnion] = [] - if isinstance(content, str): - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=[OCITextContentPart(text=content)], - toolCalls=None, - toolCallId=None, - ) - - # content is a list of content items: - # [ - # {"type": "text", "text": "Hello"}, - # {"type": "image_url", "image_url": "https://example.com/image.png"} - # ] - for content_item in content: - if not isinstance(content_item, dict): - raise Exception("Each content item must be a dictionary") - - type = content_item.get("type") - if not isinstance(type, str): - raise Exception("Prop `type` is not a string") - - if type not in ["text", "image_url"]: - raise Exception(f"Prop `{type}` is not supported") - - if type == "text": - text = content_item.get("text") - if not isinstance(text, str): - raise Exception("Prop `text` is not a string") - new_content.append(OCITextContentPart(text=text)) - - elif type == "image_url": - image_url = content_item.get("image_url") - # Handle both OpenAI format (object with url) and string format - if isinstance(image_url, dict): - image_url = image_url.get("url") - if not isinstance(image_url, str): - raise Exception( - "Prop `image_url` must be a string or an object with a `url` property" - ) - new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url))) - - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=new_content, - toolCalls=None, - toolCallId=None, - ) - - -def adapt_messages_to_generic_oci_standard_tool_call( - role: str, tool_calls: list -) -> OCIMessage: - tool_calls_formated = [] - for tool_call in tool_calls: - if not isinstance(tool_call, dict): - raise Exception("Each tool call must be a dictionary") - - if tool_call.get("type") != "function": - raise Exception("OCI only supports function tools") - - tool_call_id = tool_call.get("id") - if not isinstance(tool_call_id, str): - raise Exception("Prop `id` is not a string") - - tool_function = tool_call.get("function") - if not isinstance(tool_function, dict): - raise Exception("Prop `function` is not a dictionary") - - function_name = tool_function.get("name") - if not isinstance(function_name, str): - raise Exception("Prop `name` is not a string") - - arguments = tool_call["function"].get("arguments", "{}") - if not isinstance(arguments, str): - raise Exception("Prop `arguments` is not a string") - - # tool_calls_formated.append(OCIToolCall( - # id=tool_call_id, - # type="FUNCTION", - # function=OCIFunction( - # name=function_name, - # arguments=arguments - # ) - # )) - - tool_calls_formated.append( - OCIToolCall( - id=tool_call_id, - type="FUNCTION", - name=function_name, - arguments=arguments, - ) - ) - - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=None, - toolCalls=tool_calls_formated, - toolCallId=None, - ) - - -def adapt_messages_to_generic_oci_standard_tool_response( - role: str, tool_call_id: str, content: str -) -> OCIMessage: - return OCIMessage( - role=open_ai_to_generic_oci_role_map[role], - content=[OCITextContentPart(text=content)], - toolCalls=None, - toolCallId=tool_call_id, - ) - - -def adapt_messages_to_generic_oci_standard( - messages: List[AllMessageValues], -) -> List[OCIMessage]: - new_messages = [] - for message in messages: - role = message["role"] - content = message.get("content") - tool_calls = message.get("tool_calls") - tool_call_id = message.get("tool_call_id") - - if role == "assistant" and tool_calls is not None: - if not isinstance(tool_calls, list): - raise Exception("Prop `tool_calls` must be a list of tool calls") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls) - ) - - elif role in ["system", "user", "assistant"] and content is not None: - if not isinstance(content, (str, list)): - raise Exception( - "Prop `content` must be a string or a list of content items" - ) - new_messages.append( - adapt_messages_to_generic_oci_standard_content_message(role, content) - ) - - elif role == "tool": - if not isinstance(tool_call_id, str): - raise Exception("Prop `tool_call_id` is required and must be a string") - if not isinstance(content, str): - raise Exception("Prop `content` is not a string") - new_messages.append( - adapt_messages_to_generic_oci_standard_tool_response( - role, tool_call_id, content - ) - ) - - return new_messages - - -def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors): - new_tools = [] - for tool in tools: - if tool["type"] != "function": - raise Exception("OCI only supports function tools") - - tool_function = tool.get("function") - if not isinstance(tool_function, dict): - raise Exception("Prop `function` is not a dictionary") - - new_tool = OCIToolDefinition( - type="FUNCTION", - name=tool_function.get("name"), - description=tool_function.get("description", ""), - parameters=tool_function.get("parameters", {}), - ) - new_tools.append(new_tool) - - return new_tools - - -def adapt_tools_to_openai_standard( - tools: List[OCIToolCall], -) -> List[ChatCompletionMessageToolCall]: - new_tools = [] - for tool in tools: - new_tool = ChatCompletionMessageToolCall( - id=tool.id, - type="function", - function={ - "name": tool.name, - "arguments": tool.arguments, - }, - ) - new_tools.append(new_tool) - return new_tools - - class OCIStreamWrapper(CustomStreamWrapper): - """ - Custom stream wrapper for OCI responses. - This class is used to handle streaming responses from OCI's API. - """ + """Custom stream wrapper that dispatches OCI SSE chunks to the correct handler.""" - def __init__( - self, - **kwargs: Any, - ): + def __init__(self, **kwargs: Any): super().__init__(**kwargs) + # Tracks whether any prior Cohere chunk in this stream has emitted + # tool calls. The Cohere handler uses this to decide whether the + # terminal consolidation chunk's tool calls are duplicates (suppress) + # or the only copy of the tool calls (pass through). + self._cohere_tool_calls_emitted = False + # Analogous flag for text content. Lets the Cohere handler distinguish + # the common case (prior deltas already streamed the text, so the + # terminal chunk's text is a duplicate to suppress) from the degenerate + # single-event case (terminal chunk carries the only copy of the text). + self._cohere_text_emitted = False - def chunk_creator(self, chunk: Any): + def chunk_creator(self, chunk: Any) -> ModelResponseStream: if not isinstance(chunk, str): raise ValueError(f"Chunk is not a string: {chunk}") if not chunk.startswith("data:"): raise ValueError(f"Chunk does not start with 'data:': {chunk}") - dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON - - # Check if this is a Cohere stream chunk - if "apiFormat" in dict_chunk and dict_chunk.get("apiFormat") == "COHERE": - return self._handle_cohere_stream_chunk(dict_chunk) - else: - return self._handle_generic_stream_chunk(dict_chunk) - - def _handle_cohere_stream_chunk(self, dict_chunk: dict): - """Handle Cohere-specific streaming chunks.""" try: - typed_chunk = CohereStreamChunk(**dict_chunk) - except TypeError as e: - raise ValueError(f"Chunk cannot be casted to CohereStreamChunk: {str(e)}") + dict_chunk = json.loads(chunk[5:]) + except json.JSONDecodeError as e: + raise OCIError( + status_code=500, + message=f"Chunk cannot be parsed as JSON: {str(e)}", + ) - if typed_chunk.index is None: - typed_chunk.index = 0 + if dict_chunk.get("apiFormat") == "COHERE": + result = handle_cohere_stream_chunk( + dict_chunk, + prior_tool_calls_emitted=self._cohere_tool_calls_emitted, + prior_text_emitted=self._cohere_text_emitted, + ) + if not self._cohere_tool_calls_emitted: + for choice in result.choices: + if getattr(choice.delta, "tool_calls", None) is not None: + self._cohere_tool_calls_emitted = True + break + if not self._cohere_text_emitted: + for choice in result.choices: + if getattr(choice.delta, "content", None): + self._cohere_text_emitted = True + break + return result + return handle_generic_stream_chunk(dict_chunk) - # Extract text content - text = typed_chunk.text or "" - # Map finish reason to standard format - finish_reason = typed_chunk.finishReason - if finish_reason == "COMPLETE": - finish_reason = "stop" - elif finish_reason == "MAX_TOKENS": - finish_reason = "length" - elif finish_reason is None: - finish_reason = None - else: - finish_reason = "stop" - - # For Cohere, we don't have tool calls in the streaming format - tool_calls = None - - return ModelResponseStream( - choices=[ - StreamingChoices( - index=typed_chunk.index if typed_chunk.index else 0, - delta=Delta( - content=text, - tool_calls=tool_calls, - provider_specific_fields=None, - thinking_blocks=None, - reasoning_content=None, - ), - finish_reason=finish_reason, - ) - ] - ) - - def _handle_generic_stream_chunk(self, dict_chunk: dict): - """Handle generic OCI streaming chunks.""" - # Fix missing required fields in tool calls before Pydantic validation - # OCI streams tool calls progressively, so early chunks may be missing required fields - if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"): - for tool_call in dict_chunk["message"]["toolCalls"]: - if "arguments" not in tool_call: - tool_call["arguments"] = "" - if "id" not in tool_call: - tool_call["id"] = "" - if "name" not in tool_call: - tool_call["name"] = "" - - try: - typed_chunk = OCIStreamChunk(**dict_chunk) - except TypeError as e: - raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}") - - if typed_chunk.index is None: - typed_chunk.index = 0 - - text = "" - if typed_chunk.message and typed_chunk.message.content: - for item in typed_chunk.message.content: - if isinstance(item, OCITextContentPart): - text += item.text - elif isinstance(item, OCIImageContentPart): - raise ValueError( - "OCI does not support image content in streaming responses" - ) - else: - raise ValueError( - f"Unsupported content type in OCI response: {item.type}" - ) - - tool_calls = None - if typed_chunk.message and typed_chunk.message.toolCalls: - tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls) - - return ModelResponseStream( - choices=[ - StreamingChoices( - index=typed_chunk.index if typed_chunk.index else 0, - delta=Delta( - content=text, - tool_calls=( - [tool.model_dump() for tool in tool_calls] - if tool_calls - else None - ), - provider_specific_fields=None, # OCI does not have provider specific fields in the response - thinking_blocks=None, # OCI does not have thinking blocks in the response - reasoning_content=None, # OCI does not have reasoning content in the response - ), - finish_reason=typed_chunk.finishReason, - ) - ] - ) +__all__ = [ + "OCIChatConfig", + "OCIStreamWrapper", + "OCIRequestWrapper", + "OCI_API_VERSION", + "STREAMING_TIMEOUT", + "get_vendor_from_model", + "version", +] diff --git a/litellm/llms/oci/common_utils.py b/litellm/llms/oci/common_utils.py index 661a6c89e4..8785b1548a 100644 --- a/litellm/llms/oci/common_utils.py +++ b/litellm/llms/oci/common_utils.py @@ -1,9 +1,42 @@ -from typing import Optional +import base64 +import hashlib +import json +import os +import re +from dataclasses import dataclass +from email.utils import formatdate +from typing import Any, Dict, Optional, Protocol, Tuple +from urllib.parse import urlparse import httpx from litellm.llms.base_llm.chat.transformation import BaseLLMException +try: + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding, rsa + + _CRYPTOGRAPHY_AVAILABLE = True +except ImportError: + _CRYPTOGRAPHY_AVAILABLE = False + +try: + from litellm._version import version as _litellm_version +except ImportError: + _litellm_version = "0.0.0" + + +# OCI GenAI REST API version — stable since service launch, unlikely to change +OCI_API_VERSION = "20231130" + + +def _require_cryptography() -> None: + if not _CRYPTOGRAPHY_AVAILABLE: + raise ImportError( + "cryptography package is required for OCI authentication. " + "Please install it with: pip install cryptography" + ) + class OCIError(BaseLLMException): def __init__( @@ -17,3 +50,520 @@ class OCIError(BaseLLMException): message=message, headers=headers, ) + + +# --------------------------------------------------------------------------- +# OCI signing protocol and helpers +# --------------------------------------------------------------------------- + + +class OCISignerProtocol(Protocol): + """ + Protocol for OCI request signers (e.g., oci.signer.Signer). + + Compatible with the OCI Python SDK's Signer class. + See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html + """ + + def do_request_sign( + self, request: Any, *, enforce_content_headers: bool = False + ) -> None: + pass + + +@dataclass +class OCIRequestWrapper: + """ + Wrapper for HTTP requests compatible with OCI signer interface. + + Wraps request data in the format expected by OCI SDK signers, which require + objects with method, url, headers, body, and path_url attributes. + """ + + method: str + url: str + headers: dict + body: bytes + + @property + def path_url(self) -> str: + """Returns the path + query string for OCI signing.""" + parsed = urlparse(self.url) + return parsed.path + ("?" + parsed.query if parsed.query else "") + + +def sha256_base64(data: bytes) -> str: + # SHA-256 is used here to compute the x-content-sha256 header required by the + # OCI HTTP signing specification (RSA-SHA256 request signing), not for password + # or secret hashing. This is the correct and mandated algorithm for this purpose. + # See: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm + # + # ``usedforsecurity=False`` declares non-security intent to static analyzers + # (CodeQL ``py/weak-sensitive-data-hashing``) — without it the request body + # gets flagged as "password-like data" via taint tracking. + digest = hashlib.sha256(data, usedforsecurity=False).digest() # noqa: S324 + return base64.b64encode(digest).decode() + + +def build_signature_string( + method: str, path: str, headers: dict, signed_headers: list +) -> str: + lines = [] + for header in signed_headers: + if header == "(request-target)": + value = f"{method.lower()} {path}" + else: + value = headers[header] + lines.append(f"{header}: {value}") + return "\n".join(lines) + + +def load_private_key_from_str(key_str: str) -> Any: + _require_cryptography() + key = serialization.load_pem_private_key( # type: ignore[union-attr] + key_str.encode("utf-8"), + password=None, + ) + if not isinstance(key, rsa.RSAPrivateKey): # type: ignore[union-attr] + raise TypeError( + "The provided private key is not an RSA key, which is required for OCI signing." + ) + return key + + +def load_private_key_from_file(file_path: str) -> Any: + """Loads a private key from a file path.""" + try: + with open(file_path, "r", encoding="utf-8") as f: + key_str = f.read().strip() + except FileNotFoundError: + raise FileNotFoundError(f"Private key file not found: {file_path}") + except OSError as e: + raise OSError(f"Failed to read private key file '{file_path}': {e}") from e + + if not key_str: + raise ValueError(f"Private key file is empty: {file_path}") + + return load_private_key_from_str(key_str) + + +# --------------------------------------------------------------------------- +# Env-var credential resolution +# --------------------------------------------------------------------------- + +_OCI_REGION_ENV = "OCI_REGION" +_OCI_USER_ENV = "OCI_USER" +_OCI_FINGERPRINT_ENV = "OCI_FINGERPRINT" +_OCI_TENANCY_ENV = "OCI_TENANCY" +_OCI_KEY_FILE_ENV = "OCI_KEY_FILE" +_OCI_KEY_ENV = "OCI_KEY" +_OCI_COMPARTMENT_ID_ENV = "OCI_COMPARTMENT_ID" + + +def resolve_oci_credentials(optional_params: dict) -> dict: + """ + Merge OCI credentials from optional_params (explicit, always wins) and + environment variables (fallback). + + Returns a dict with resolved values for: + oci_region, oci_user, oci_fingerprint, oci_tenancy, + oci_key, oci_key_file, oci_compartment_id + """ + return { + "oci_region": optional_params.get("oci_region") + or os.environ.get(_OCI_REGION_ENV) + or "us-ashburn-1", + "oci_user": optional_params.get("oci_user") or os.environ.get(_OCI_USER_ENV), + "oci_fingerprint": optional_params.get("oci_fingerprint") + or os.environ.get(_OCI_FINGERPRINT_ENV), + "oci_tenancy": optional_params.get("oci_tenancy") + or os.environ.get(_OCI_TENANCY_ENV), + "oci_key": optional_params.get("oci_key") or os.environ.get(_OCI_KEY_ENV), + "oci_key_file": optional_params.get("oci_key_file") + or os.environ.get(_OCI_KEY_FILE_ENV), + "oci_compartment_id": optional_params.get("oci_compartment_id") + or os.environ.get(_OCI_COMPARTMENT_ID_ENV), + } + + +_OCI_REGION_RE = re.compile(r"^[a-z][a-z0-9-]{0,30}[a-z0-9]$") +_OCI_ACTION_PATH_RE = re.compile(rf"/{OCI_API_VERSION}/actions/[^/?#]+/?$") + + +def get_oci_base_url(optional_params: dict, api_base: Optional[str] = None) -> str: + """Return the OCI inference base URL, respecting any explicit api_base override. + + If ``api_base`` already ends with a fully-formed OCI action path + (``/{OCI_API_VERSION}/actions/``), that suffix is stripped so callers + can append their own action path without producing a doubled URL. + """ + if api_base: + return _OCI_ACTION_PATH_RE.sub("", api_base).rstrip("/") + creds = resolve_oci_credentials(optional_params) + region = creds["oci_region"] + if not isinstance(region, str) or not _OCI_REGION_RE.match(region): + raise OCIError( + status_code=400, + message=( + f"Invalid OCI region {region!r}: must match " + "^[a-z][a-z0-9-]{0,30}[a-z0-9]$ (e.g. 'us-ashburn-1')." + ), + ) + return f"https://inference.generativeai.{region}.oci.oraclecloud.com" + + +# --------------------------------------------------------------------------- +# Signing implementations (shared by chat, embed, and rerank configs) +# --------------------------------------------------------------------------- + + +def sign_with_oci_signer( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, +) -> Tuple[dict, bytes]: + """Sign a request using an OCI SDK Signer object passed in optional_params.""" + oci_signer = optional_params.get("oci_signer") + body = json.dumps(request_data).encode("utf-8") + method = str(optional_params.get("method", "POST")).upper() + + if method not in {"POST", "GET", "PUT", "DELETE", "PATCH"}: + raise ValueError(f"Unsupported HTTP method: {method}") + + prepared_headers = {**headers} + prepared_headers.setdefault("content-type", "application/json") + prepared_headers.setdefault("content-length", str(len(body))) + + request_wrapper = OCIRequestWrapper( + method=method, url=api_base, headers=prepared_headers, body=body + ) + + if oci_signer is None: + raise ValueError("oci_signer cannot be None when calling sign_with_oci_signer") + + try: + oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True) + except Exception as e: + raise OCIError( + status_code=500, + message=( + f"Failed to sign request with provided oci_signer: {str(e)}. " + "The signer must implement the OCI SDK Signer interface with a " + "do_request_sign(request, enforce_content_headers=True) method. " + "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" + ), + ) from e + + headers.update(request_wrapper.headers) + return headers, body + + +def sign_with_manual_credentials( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, +) -> Tuple[dict, bytes]: + """Sign a request using manually provided OCI credentials (user/fingerprint/tenancy/key).""" + creds = resolve_oci_credentials(optional_params) + oci_user = creds["oci_user"] + oci_fingerprint = creds["oci_fingerprint"] + oci_tenancy = creds["oci_tenancy"] + oci_key = creds["oci_key"] + oci_key_file = creds["oci_key_file"] + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not (oci_key or oci_key_file) + ): + raise OCIError( + status_code=401, + message=( + "Missing required OCI credentials: oci_user, oci_fingerprint, oci_tenancy, " + "and at least one of oci_key or oci_key_file. " + "These can also be supplied via environment variables: " + f"{_OCI_USER_ENV}, {_OCI_FINGERPRINT_ENV}, {_OCI_TENANCY_ENV}, {_OCI_KEY_ENV} (or {_OCI_KEY_FILE_ENV}). " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + + method = str(optional_params.get("method", "POST")).upper() + body = json.dumps(request_data).encode("utf-8") + parsed = urlparse(api_base) + path = parsed.path or "/" + host = parsed.netloc + + date = formatdate(usegmt=True) + content_type = headers.get("content-type", "application/json") + content_length = str(len(body)) + x_content_sha256 = sha256_base64(body) + + headers_to_sign: Dict[str, str] = { + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + + signed_header_names = [ + "date", + "(request-target)", + "host", + "content-length", + "content-type", + "x-content-sha256", + ] + signing_string = build_signature_string( + method, path, headers_to_sign, signed_header_names + ) + + _require_cryptography() + + # Resolve the private key — prefer inline PEM content over file path + oci_key_content: Optional[str] = None + if oci_key: + if not isinstance(oci_key, str): + raise OCIError( + status_code=400, + message=( + f"oci_key must be a string containing the PEM private key content. " + f"Got type: {type(oci_key).__name__}" + ), + ) + oci_key_content = oci_key.replace("\\n", "\n").replace("\r\n", "\n") + + private_key = ( + load_private_key_from_str(oci_key_content) + if oci_key_content + else load_private_key_from_file(oci_key_file) if oci_key_file else None + ) + + if private_key is None: + raise OCIError( + status_code=400, + message="Private key is required for OCI authentication. Provide either oci_key or oci_key_file.", + ) + + signature = private_key.sign( + signing_string.encode("utf-8"), + padding.PKCS1v15(), # type: ignore[union-attr] + hashes.SHA256(), # type: ignore[union-attr] + ) + signature_b64 = base64.b64encode(signature).decode() + + key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}" + authorization = ( + 'Signature version="1",' + f'keyId="{key_id}",' + 'algorithm="rsa-sha256",' + f'headers="{" ".join(signed_header_names)}",' + f'signature="{signature_b64}"' + ) + + headers.update( + { + "authorization": authorization, + "date": date, + "host": host, + "content-type": content_type, + "content-length": content_length, + "x-content-sha256": x_content_sha256, + } + ) + return headers, body + + +def sign_oci_request( + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, +) -> Tuple[dict, bytes]: + """ + Route to the appropriate OCI signing method based on what credentials are present. + + If ``oci_signer`` is in optional_params, use the OCI SDK signer object. + Otherwise use manual RSA-SHA256 signing with explicit credentials (which can + also be supplied via OCI_* environment variables). + + Returns: + Tuple of (signed_headers, signed_body_bytes) + """ + if optional_params.get("oci_signer") is not None: + return sign_with_oci_signer(headers, optional_params, request_data, api_base) + return sign_with_manual_credentials( + headers, optional_params, request_data, api_base + ) + + +def validate_oci_environment( + headers: dict, + optional_params: dict, + api_key: Optional[str] = None, +) -> dict: + """ + Populate common OCI request headers (content-type, user-agent). + + Full credential validation is deferred to signing time so that credentials + supplied via environment variables are resolved at call time rather than + at construction time. + """ + headers.setdefault("content-type", "application/json") + headers.setdefault("user-agent", f"litellm/{_litellm_version}") + return headers + + +# --------------------------------------------------------------------------- +# JSON schema utilities for OCI tool definitions +# +# OCI Generative AI does not support JSON Schema extensions ($ref, $defs, +# anyOf). Pydantic v2 emits all three for models with Optional fields or +# nested schemas. The helpers below are ported from the official +# langchain-oracle reference implementation so that tool schemas are always +# valid before they reach the OCI endpoint. +# --------------------------------------------------------------------------- + +# Mapping from JSON Schema type names to Python type names, as expected by +# the OCI Cohere API's CohereParameterDefinition.type field. +OCI_JSON_TO_PYTHON_TYPES: Dict[str, str] = { + "string": "str", + "number": "float", + "boolean": "bool", + "integer": "int", + "array": "List", + "object": "Dict", + "any": "any", +} + + +def resolve_oci_schema_refs(schema: Dict[str, Any]) -> Dict[str, Any]: + """Inline all ``$ref``/``$defs`` references — OCI does not support JSON Schema ``$ref``.""" + defs = schema.get("$defs", {}) + resolving_stack: set = set() + + def _resolve(obj: Any) -> Any: + if isinstance(obj, dict): + if "$ref" in obj: + ref = obj["$ref"] + if ref.startswith("#/$defs/"): + key = ref.split("/")[-1] + if key in resolving_stack: + return {"type": "object"} # break cycles + resolving_stack.add(key) + try: + return _resolve(defs.get(key, obj)) + finally: + resolving_stack.discard(key) + return obj # external $ref — leave unchanged + return {k: _resolve(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_resolve(item) for item in obj] + return obj + + resolved = _resolve(schema) + if isinstance(resolved, dict): + resolved.pop("$defs", None) + return resolved + + +def resolve_oci_schema_anyof(obj: Any) -> Any: + """Resolve Pydantic v2 ``Optional[T]`` → ``anyOf`` patterns. + + Pydantic v2 emits ``{"anyOf": [{"type": "T"}, {"type": "null"}]}`` for + ``Optional[T]``. OCI models don't understand ``anyOf``, so we pick the + first non-null branch and merge top-level metadata into it. + """ + if isinstance(obj, dict): + if "anyOf" in obj and "type" not in obj: + non_null = [ + t + for t in obj["anyOf"] + if not (isinstance(t, dict) and t.get("type") == "null") + ] + if non_null: + resolved = {**obj, **non_null[0]} + resolved.pop("anyOf", None) + return resolve_oci_schema_anyof(resolved) + return {k: resolve_oci_schema_anyof(v) for k, v in obj.items()} + if isinstance(obj, list): + return [resolve_oci_schema_anyof(item) for item in obj] + return obj + + +def sanitize_oci_schema(schema: Any) -> Any: + """Recursively remove OCI-incompatible fields from a JSON schema. + + Strips ``title`` keys, removes ``None``-valued ``default`` entries, + normalises ``type: [T, "null"]`` list types, and ensures arrays carry an + ``items`` definition. + """ + if isinstance(schema, list): + return [sanitize_oci_schema(item) for item in schema] + if not isinstance(schema, dict): + return schema + + sanitized: Dict[str, Any] = {} + for key, value in schema.items(): + if key == "title": + continue + if key == "default" and value is None: + continue + if key == "type": + if value == "any": + sanitized[key] = "object" + continue + if isinstance(value, list): + non_null = [t for t in value if t != "null"] + sanitized[key] = non_null[0] if non_null else "string" + continue + sanitized[key] = sanitize_oci_schema(value) + + if sanitized.get("type") == "array" and "items" not in sanitized: + sanitized["items"] = {"type": "object"} + + required = sanitized.get("required") + properties = sanitized.get("properties") + if "required" in sanitized: + if isinstance(required, list) and isinstance(properties, dict): + sanitized["required"] = [ + f for f in required if isinstance(f, str) and f in properties + ] + elif not isinstance(required, list): + sanitized["required"] = [] + + return sanitized + + +def enrich_cohere_param_description( + description: str, param_schema: Dict[str, Any] +) -> str: + """Embed schema constraints into a Cohere parameter description. + + ``CohereParameterDefinition`` only has ``type``, ``description``, and + ``isRequired``. Rich constraints (``enum``, ``format``, ``minimum``, + ``maximum``, ``pattern``) are appended to the description string so the + model can still see and respect them. + """ + parts = [description] if description else [] + if "enum" in param_schema: + parts.append(f"Allowed values: {param_schema['enum']}") + if "format" in param_schema: + parts.append(f"Format: {param_schema['format']}") + if "minimum" in param_schema or "maximum" in param_schema: + range_parts = [] + if "minimum" in param_schema: + range_parts.append(f"min={param_schema['minimum']}") + if "maximum" in param_schema: + range_parts.append(f"max={param_schema['maximum']}") + parts.append(f"Range: {', '.join(range_parts)}") + if "pattern" in param_schema: + parts.append(f"Pattern: {param_schema['pattern']}") + return ". ".join(parts) if parts else "" diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py index 1dcd8c5213..6cfa85b4bc 100644 --- a/litellm/llms/oci/embed/transformation.py +++ b/litellm/llms/oci/embed/transformation.py @@ -1,8 +1,14 @@ """ -OCI Generative AI Embedding Configuration +OCI Generative AI — Embedding transformation. -Supports embedding models available on Oracle Cloud Infrastructure Generative AI service. -Uses the same authentication mechanisms as OCI chat (manual signing or OCI SDK Signer). +Endpoint: POST /20231130/actions/embedText +Supported models: cohere.embed-english-v3.0, cohere.embed-multilingual-v3.0, +cohere.embed-v4.0, and all other Cohere embed variants available on OCI +(including dedicated endpoints). + +Authentication follows the same RSA-SHA256 / OCI SDK signer pattern as chat. +The base handler (base_llm_http_handler.embedding) calls sign_request after +building the body, so signing happens automatically. Supported models: - cohere.embed-english-v3.0 @@ -10,25 +16,45 @@ Supported models: - cohere.embed-multilingual-v3.0 - cohere.embed-multilingual-light-v3.0 - cohere.embed-english-image-v3.0 -- cohere.embed-english-light-image-v3.0 -- cohere.embed-multilingual-light-image-v3.0 +- cohere.embed-multilingual-image-v3.0 - cohere.embed-v4.0 Reference: https://docs.oracle.com/en-us/iaas/api/#/en/generative-ai-inference/latest/EmbedTextResult/EmbedText """ -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union import httpx -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.llms.oci.chat.transformation import OCIChatConfig -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + get_oci_base_url, + resolve_oci_credentials, + sign_oci_request, + validate_oci_environment, +) +from litellm.types.llms.oci import ( + OCIEmbedRequest, + OCIEmbedResponse, + OCIServingMode, +) from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +# OCI sends up to 96 texts per embedText request (Cohere limit). +OCI_EMBED_BATCH_LIMIT = 96 + # Input type mapping from OpenAI conventions to OCI/Cohere conventions _INPUT_TYPE_MAP = { "search_document": "SEARCH_DOCUMENT", @@ -38,65 +64,43 @@ _INPUT_TYPE_MAP = { } -class OCIEmbeddingConfig(BaseEmbeddingConfig): +class OCIEmbedConfig(BaseEmbeddingConfig): """ - Configuration for OCI Generative AI Embedding API. + Transformation config for OCI Generative AI embeddings. - The OCI embedding endpoint uses the Cohere embed models hosted on OCI. - Authentication is handled via OCI request signing (manual credentials or OCI SDK Signer). + Supports both text and (on cohere.embed-v4.0) multimodal inputs. - Usage: - ```python - import litellm + Authentication — same two modes as chat: + - **OCI SDK signer**: pass ``oci_signer`` in optional_params. + - **Manual RSA-SHA256**: pass ``oci_user``, ``oci_fingerprint``, ``oci_tenancy``, + and ``oci_key`` or ``oci_key_file``, or set the corresponding ``OCI_*`` env vars. - response = litellm.embedding( - model="oci/cohere.embed-english-v3.0", - input=["Hello world", "Goodbye world"], - oci_compartment_id="ocid1.compartment.oc1..xxx", - oci_region="us-ashburn-1", - oci_user="ocid1.user.oc1..xxx", - oci_fingerprint="xx:xx:xx:xx", - oci_tenancy="ocid1.tenancy.oc1..xxx", - oci_key_file="~/.oci/key.pem", - ) - ``` + Required call-time params (via optional_params or env vars): + - ``oci_compartment_id`` / ``OCI_COMPARTMENT_ID`` + - ``oci_region`` / ``OCI_REGION`` (default: ``us-ashburn-1``) + + Optional call-time params: + - ``oci_serving_mode``: ``"ON_DEMAND"`` (default) or ``"DEDICATED"`` + - ``oci_endpoint_id``: endpoint OCID for dedicated serving mode + - ``input_type``: ``SEARCH_DOCUMENT``, ``SEARCH_QUERY``, ``CLASSIFICATION``, ``CLUSTERING`` + - ``truncate``: ``NONE``, ``START``, or ``END`` (default ``END``) + - ``dimensions``: output embedding dimensions (cohere.embed-v4.0+) """ - def __init__(self) -> None: - # We reuse OCIChatConfig for signing logic - self._chat_config = OCIChatConfig() - - def get_complete_url( - self, - api_base: Optional[str], - api_key: Optional[str], - model: str, - optional_params: dict, - litellm_params: dict, - stream: Optional[bool] = None, - ) -> str: - if api_base: - return api_base - - oci_region = optional_params.get("oci_region", "us-ashburn-1") - return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/embedText" - - def get_supported_openai_params(self, model: str) -> list: - return [ - "dimensions", - ] + def get_supported_openai_params(self, model: str) -> List[str]: + return ["dimensions"] def map_openai_params( self, non_default_params: dict, optional_params: dict, model: str, - drop_params: bool, + drop_params: bool = False, ) -> dict: - # Note: OCI Cohere embed does not support custom dimensions natively, - # but we pass it through in case future models support it - if "dimensions" in non_default_params: - optional_params["dimensions"] = non_default_params["dimensions"] + for key, value in non_default_params.items(): + if key == "dimensions": + # OCI API uses outputDimensions (cohere.embed-v4.0+) + optional_params["outputDimensions"] = value return optional_params def validate_environment( @@ -109,49 +113,42 @@ class OCIEmbeddingConfig(BaseEmbeddingConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - """ - Validate OCI credentials for embedding requests. - Supports both OCI SDK Signer and manual credential signing. - """ - oci_signer = optional_params.get("oci_signer") - oci_region = optional_params.get("oci_region", "us-ashburn-1") - - api_base = ( - api_base - or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" - ) - - if oci_signer is None: - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - oci_compartment_id = optional_params.get("oci_compartment_id") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - or not oci_compartment_id - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " - "and at least one of oci_key or oci_key_file. " - "Alternatively, provide an oci_signer object from the OCI SDK." + if optional_params.get("oci_signer") is None: + creds = resolve_oci_credentials(optional_params) + missing = [ + k + for k in ( + "oci_user", + "oci_fingerprint", + "oci_tenancy", + "oci_compartment_id", ) + if not creds.get(k) + ] + if missing or not (creds.get("oci_key") or creds.get("oci_key_file")): + raise OCIError( + status_code=401, + message=( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, " + "oci_compartment_id and at least one of oci_key or oci_key_file. " + "These can be supplied via optional_params or via OCI_USER, OCI_FINGERPRINT, " + "OCI_TENANCY, OCI_COMPARTMENT_ID, OCI_KEY_FILE environment variables. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ), + ) + return validate_oci_environment(headers, optional_params, api_key) - from litellm.llms.custom_httpx.http_handler import version - - headers.update( - { - "content-type": "application/json", - "user-agent": f"litellm/{version}", - } - ) - - return headers + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + base = get_oci_base_url(optional_params, api_base or litellm.api_base) + return f"{base}/{OCI_API_VERSION}/actions/embedText" def sign_request( self, @@ -163,9 +160,8 @@ class OCIEmbeddingConfig(BaseEmbeddingConfig): model: Optional[str] = None, stream: Optional[bool] = None, fake_stream: Optional[bool] = None, - ): - """Delegate to OCIChatConfig's signing logic.""" - return self._chat_config.sign_request( + ) -> Tuple[dict, bytes]: + return sign_oci_request( headers=headers, optional_params=optional_params, request_data=request_data, @@ -182,91 +178,74 @@ class OCIEmbeddingConfig(BaseEmbeddingConfig): input: AllEmbeddingInputValues, optional_params: dict, headers: dict, - api_base: Optional[str] = None, ) -> dict: - """ - Transform the embedding request to OCI format. - - OCI embedText API expects: - { - "compartmentId": "...", - "servingMode": {"servingType": "ON_DEMAND", "modelId": "..."}, - "inputs": ["text1", "text2"], - "truncate": "END", - "inputType": "SEARCH_DOCUMENT" - } - """ - oci_compartment_id = optional_params.get("oci_compartment_id") - if not oci_compartment_id: - raise Exception( - "kwarg `oci_compartment_id` is required for OCI embedding requests" + creds = resolve_oci_credentials(optional_params) + compartment_id = creds["oci_compartment_id"] + if not compartment_id: + raise OCIError( + status_code=400, + message=( + "oci_compartment_id is required for OCI embedding requests. " + "Pass it as optional_params or set the OCI_COMPARTMENT_ID env var." + ), ) - # Build serving mode - oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND") - if oci_serving_mode == "DEDICATED": - oci_endpoint_id = optional_params.get("oci_endpoint_id", model) - serving_mode = { - "servingType": "DEDICATED", - "endpointId": oci_endpoint_id, - } - else: - serving_mode = { - "servingType": "ON_DEMAND", - "modelId": model, - } - - # Normalize input to list of strings + # Normalise input to a flat list of strings if isinstance(input, str): - inputs = [input] + texts = [input] elif isinstance(input, list): - inputs = [] + texts = [] for item in input: - if isinstance(item, str): - inputs.append(item) - elif isinstance(item, list): - raise ValueError( - "OCI embedding does not support token-array inputs. " - "Please convert token lists to strings before calling embedding()." + if isinstance(item, list): + raise OCIError( + status_code=400, + message=( + "OCI embedText does not support token-array inputs. " + "Convert token lists to strings before calling embedding()." + ), ) - else: - inputs.append(str(item)) + texts.append(item if isinstance(item, str) else str(item)) else: - inputs = [str(input)] + texts = [str(input)] - # Build request data — OCI embedText API expects inputs, truncate, - # and inputType at the top level alongside compartmentId and servingMode - request_data: Dict[str, Any] = { - "compartmentId": oci_compartment_id, - "servingMode": serving_mode, - "inputs": inputs, - "truncate": optional_params.get("truncate", "END"), - } + if len(texts) > OCI_EMBED_BATCH_LIMIT: + raise OCIError( + status_code=400, + message=( + f"OCI embedText accepts at most {OCI_EMBED_BATCH_LIMIT} inputs per request " + f"(got {len(texts)}). Batch your requests." + ), + ) - # Map input_type if provided + serving_mode_type = optional_params.get("oci_serving_mode", "ON_DEMAND").upper() + if serving_mode_type not in {"ON_DEMAND", "DEDICATED"}: + raise OCIError( + status_code=400, + message="oci_serving_mode must be 'ON_DEMAND' or 'DEDICATED'.", + ) + + if serving_mode_type == "DEDICATED": + endpoint_id = optional_params.get("oci_endpoint_id", model) + serving_mode = OCIServingMode( + servingType="DEDICATED", endpointId=endpoint_id + ) + else: + serving_mode = OCIServingMode(servingType="ON_DEMAND", modelId=model) + + # Map input_type from OpenAI convention to OCI/Cohere convention input_type = optional_params.get("input_type") if input_type: - mapped_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) - request_data["inputType"] = mapped_type + input_type = _INPUT_TYPE_MAP.get(input_type.lower(), input_type.upper()) - # Sign the request using the same URL the HTTP handler will POST to - signing_url = self.get_complete_url( - api_base=api_base, - api_key=None, - model=model, - optional_params=optional_params, - litellm_params={}, + request = OCIEmbedRequest( + compartmentId=compartment_id, + servingMode=serving_mode, + inputs=texts, + inputType=input_type, + truncate=optional_params.get("truncate", "END"), + outputDimensions=optional_params.get("outputDimensions"), ) - - signed_headers, body = self.sign_request( - headers=headers, - optional_params=optional_params, - request_data=request_data, - api_base=signing_url, - ) - headers.update(signed_headers) - - return request_data + return request.model_dump(exclude_none=True) def transform_embedding_response( self, @@ -274,63 +253,57 @@ class OCIEmbeddingConfig(BaseEmbeddingConfig): raw_response: httpx.Response, model_response: EmbeddingResponse, logging_obj: LiteLLMLoggingObj, - api_key: Optional[str] = None, - request_data: dict = {}, - optional_params: dict = {}, - litellm_params: dict = {}, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, ) -> EmbeddingResponse: - """ - Transform OCI embedding response to standard EmbeddingResponse format. - - OCI response format: - { - "embeddings": [[0.1, 0.2, ...], [0.3, 0.4, ...]], - "modelId": "cohere.embed-english-v3.0", - "modelVersion": "3.0", - "inputTextTokenCounts": [5, 4] - } - """ if raw_response.status_code != 200: raise OCIError( - message=raw_response.text, status_code=raw_response.status_code, + message=raw_response.text, ) try: - raw_response_json = raw_response.json() - except Exception: + json_response = raw_response.json() + except Exception as e: raise OCIError( - message=raw_response.text, status_code=raw_response.status_code, + message=f"Failed to parse OCI embed response as JSON: {e}", ) - embeddings = raw_response_json.get("embeddings", []) - model_id = raw_response_json.get("modelId", model) - - # Build response data in OpenAI format - embedding_data = [] - for idx, embedding in enumerate(embeddings): - embedding_data.append( - { - "object": "embedding", - "index": idx, - "embedding": embedding, - } + try: + parsed = OCIEmbedResponse(**json_response) + except Exception as e: + raise OCIError( + status_code=500, + message=f"OCI embed response does not match expected schema: {e}", ) - model_response.model = model_id - model_response.data = embedding_data - model_response.object = "list" + model_response.model = parsed.modelId + model_response.data = [ + { + "object": "embedding", + "index": i, + "embedding": embedding, + } + for i, embedding in enumerate(parsed.embeddings) + ] - # Calculate token usage - input_token_counts = raw_response_json.get("inputTextTokenCounts", []) - total_tokens = sum(input_token_counts) if input_token_counts else 0 - - usage = Usage( - prompt_tokens=total_tokens, - total_tokens=total_tokens, - ) - model_response.usage = usage + if parsed.inputTextTokenCounts is not None: + # Actual OCI API returns per-input token counts — sum for total usage + total = sum(parsed.inputTextTokenCounts) + model_response.usage = Usage(prompt_tokens=total, total_tokens=total) + elif parsed.usage is not None: + # Some deployments may return a usage object directly + model_response.usage = Usage( + prompt_tokens=parsed.usage.promptTokens, + total_tokens=parsed.usage.totalTokens, + ) + else: + # Neither field returned — default to zero so downstream consumers + # can always rely on usage being populated. + model_response.usage = Usage(prompt_tokens=0, total_tokens=0) return model_response @@ -340,8 +313,8 @@ class OCIEmbeddingConfig(BaseEmbeddingConfig): status_code: int, headers: Union[dict, httpx.Headers], ) -> BaseLLMException: - return OCIError( - message=error_message, - status_code=status_code, - headers=headers if isinstance(headers, httpx.Headers) else None, - ) + return OCIError(status_code=status_code, message=error_message) + + +# Alias for backwards compatibility with any code that imports OCIEmbeddingConfig +OCIEmbeddingConfig = OCIEmbedConfig diff --git a/litellm/main.py b/litellm/main.py index e17a5ad9a4..510e6e424a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5127,6 +5127,24 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif custom_llm_provider == "oci": + if headers is None: + headers = {} + response = base_llm_http_handler.embedding( + model=model, + input=input, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + logging_obj=logging, + timeout=timeout, + model_response=EmbeddingResponse(), + optional_params=optional_params, + client=client, + aembedding=aembedding, + litellm_params=litellm_params_dict, + headers=headers, + ) elif custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat": cohere_key = ( api_key @@ -5807,22 +5825,6 @@ def embedding( # noqa: PLR0915 aembedding=aembedding, litellm_params={}, ) - elif custom_llm_provider == "oci": - response = base_llm_http_handler.embedding( - model=model, - input=input, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, - logging_obj=logging, - timeout=timeout, - model_response=EmbeddingResponse(), - optional_params=optional_params, - client=client, - aembedding=aembedding, - litellm_params=litellm_params_dict, - headers=headers, - ) elif custom_llm_provider in litellm._custom_providers: custom_handler: Optional[CustomLLM] = None for item in litellm.custom_provider_map: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6a4a5dd6a0..08c06b17dd 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26297,6 +26297,51 @@ "supports_function_calling": true, "supports_response_schema": false }, + "oci/openai.gpt-5": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/openai.gpt-5-mini": { + "input_cost_per_token": 2.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "oci/openai.gpt-5-nano": { + "input_cost_per_token": 5e-08, + "litellm_provider": "oci", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, "oci/google.gemini-2.5-pro": { "input_cost_per_token": 1.25e-06, "litellm_provider": "oci", diff --git a/litellm/router.py b/litellm/router.py index debccb0e83..2c61031da3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10764,7 +10764,7 @@ class Router: if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11028,7 +11028,7 @@ class Router: if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11202,7 +11202,7 @@ class Router: if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) @@ -11359,7 +11359,7 @@ class Router: if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: raise litellm.ServiceUnavailableError( - message=f"Model '{model}' is administratively paused. Contact your proxy admin to unblock it.", + message=f"Model '{model}' is currently paused and cannot accept requests.", model=model, llm_provider="", ) diff --git a/litellm/types/llms/oci.py b/litellm/types/llms/oci.py index e041810158..df551d8a8c 100644 --- a/litellm/types/llms/oci.py +++ b/litellm/types/llms/oci.py @@ -3,7 +3,7 @@ from __future__ import annotations from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel +from pydantic import BaseModel, SerializeAsAny OCIRoles = Literal["SYSTEM", "USER", "ASSISTANT", "TOOL"] @@ -15,7 +15,6 @@ class OCIVendors(Enum): """ COHERE = "COHERE" - GEMINI = "GEMINI" GENERIC = "GENERIC" @@ -57,7 +56,7 @@ OCIContentPartUnion = Union[OCITextContentPart, OCIImageContentPart] class OCIToolCall(BaseModel): """Represents a tool call made by the model.""" - id: str + id: Optional[str] = None # absent in some provider responses (e.g. Google via OCI) type: Literal["FUNCTION"] = "FUNCTION" name: str arguments: str # Arguments should be a JSON-serialized string @@ -96,13 +95,22 @@ class OCIChatRequestPayload(BaseModel): isStream: bool = False numGenerations: Optional[int] = None maxTokens: Optional[int] = None + # GPT-5+ on OCI rejects maxTokens and requires maxCompletionTokens. + maxCompletionTokens: Optional[int] = None temperature: Optional[float] = None topP: Optional[float] = None stop: Optional[List[str]] = None seed: Optional[int] = None frequencyPenalty: Optional[float] = None presencePenalty: Optional[float] = None + # Reasoning-token budget knob (OCI: NONE/MINIMAL/LOW/MEDIUM/HIGH). + # Honoured by GPT-5 family, Gemini 2.5, Grok reasoning variants, + # Cohere Command-A-Reasoning. Ignored by non-reasoning models. + reasoningEffort: Optional[str] = None responseFormat: Optional[Dict[str, Any]] = None + toolChoice: Optional[Union[str, Dict[str, Any]]] = None + logitBias: Optional[Dict[str, Any]] = None + logProbs: Optional[int] = None class OCIServingMode(BaseModel): @@ -141,7 +149,9 @@ class OCIResponseUsage(BaseModel): """Token usage in the OCI response.""" promptTokens: int - completionTokens: int + # completionTokens may be absent for reasoning models when all the output + # budget is consumed by reasoning tokens before any visible content is produced. + completionTokens: Optional[int] = None totalTokens: int completionTokensDetails: Optional[OCICompletionTokenDetails] = None promptTokensDetails: Optional[OCIPromptTokensDetails] = None @@ -151,7 +161,9 @@ class OCIResponseChoice(BaseModel): """A completion choice in the OCI response.""" index: int - message: OCIMessage + # message is absent when a reasoning model exhausts max_tokens in the + # reasoning phase without producing any visible content. + message: Optional[OCIMessage] = None finishReason: Optional[str] = None logprobs: Optional[Dict[str, Any]] = None @@ -203,6 +215,7 @@ class CohereStreamChunk(BaseModel): text: Optional[str] = None chatHistory: Optional[List[CohereMessage]] = None finishReason: Optional[str] = None + toolCalls: Optional[List[CohereToolCall]] = None pad: Optional[str] = None index: Optional[int] = None @@ -234,10 +247,14 @@ class CohereSystemMessage(CohereMessage): class CohereToolMessage(CohereMessage): - """Tool message in Cohere chat.""" + """Tool message in Cohere chat. + + The OCI Cohere API represents tool results via a ``toolResults`` list on the + TOOL-role history entry — not via a ``toolCallId`` string. + """ role: Literal["TOOL"] = "TOOL" - toolCallId: str + toolResults: List[CohereToolResult] class CohereParameterDefinition(BaseModel): @@ -264,10 +281,14 @@ class CohereToolCall(BaseModel): class CohereToolResult(BaseModel): - """Result of a tool call.""" + """Result of a tool call. - callId: str - result: str + Matches the OCI SDK's CohereToolResult: each result carries the originating + tool call (name + parameters) and a list of output objects. + """ + + call: CohereToolCall + outputs: List[Dict[str, Any]] class CohereResponseFormat(BaseModel): @@ -297,7 +318,11 @@ class CohereChatRequest(BaseModel): apiFormat: Literal["COHERE"] = "COHERE" # Optional fields - chatHistory: Optional[List[CohereMessage]] = None + # ``SerializeAsAny`` preserves subclass-specific fields (e.g. ``toolResults`` + # on ``CohereToolMessage``) when this request is serialized via ``model_dump``. + # Without it, Pydantic v2 would serialize each element using the declared + # ``CohereMessage`` schema and silently drop subclass fields. + chatHistory: Optional[List[SerializeAsAny[CohereMessage]]] = None maxTokens: Optional[int] = None temperature: Optional[float] = None topP: Optional[float] = None @@ -307,7 +332,10 @@ class CohereChatRequest(BaseModel): stopSequences: Optional[List[str]] = None seed: Optional[int] = None tools: Optional[List[CohereTool]] = None - toolChoice: Optional[Union[str, Dict[str, Any]]] = None + # NOTE: OCI's Cohere chat endpoint does not accept ``toolChoice`` — see + # ``OCIChatConfig.openai_to_oci_cohere_param_map`` which marks + # ``tool_choice`` as unsupported. The field is intentionally absent here + # so it isn't silently dropped or surfaced as a supported feature. responseFormat: Optional[ Union[ CohereResponseTextFormat, @@ -364,9 +392,12 @@ class CohereChatResponse(BaseModel): # Required fields text: str apiFormat: Literal["COHERE"] = "COHERE" - finishReason: Literal[ - "COMPLETE", "ERROR_TOXIC", "ERROR_LIMIT", "ERROR", "USER_CANCEL", "MAX_TOKENS" - ] + # Accept any string (with ``None`` for absent) so unknown finish reasons + # — e.g. a value OCI adds in a future API revision — degrade gracefully + # via ``handle_cohere_response``'s ``elif oci_finish_reason is not None`` + # fallback instead of crashing Pydantic validation. Mirrors + # ``CohereStreamChunk.finishReason`` which has always been ``Optional[str]``. + finishReason: Optional[str] = None # Optional fields chatHistory: Optional[List[CohereMessage]] = None @@ -394,3 +425,41 @@ class CohereChatResult(BaseModel): modelId: str modelVersion: str chatResponse: CohereChatResponse + + +# --------------------------------------------------------------------------- +# OCI Embed types +# --------------------------------------------------------------------------- + + +class OCIEmbedRequest(BaseModel): + """Request body for POST /20231130/actions/embedText.""" + + compartmentId: str + servingMode: OCIServingMode + inputs: List[str] + inputType: Optional[str] = ( + None # SEARCH_DOCUMENT | SEARCH_QUERY | CLASSIFICATION | CLUSTERING | IMAGE + ) + truncate: Optional[str] = "END" # NONE | START | END + outputDimensions: Optional[int] = ( + None # cohere.embed-v4.0+; valid: 256, 512, 1024, 1536 + ) + + +class OCIEmbedUsage(BaseModel): + promptTokens: int + totalTokens: int + + +class OCIEmbedResponse(BaseModel): + """Response body from POST /20231130/actions/embedText.""" + + id: Optional[str] = None # present in the official SDK response + embeddings: List[List[float]] + modelId: str + modelVersion: str + # OCI returns per-input token counts in inputTextTokenCounts (summed for total usage) + inputTextTokenCounts: Optional[List[int]] = None + # Some deployments may return a usage object instead + usage: Optional[OCIEmbedUsage] = None diff --git a/litellm/utils.py b/litellm/utils.py index c28a88e0f1..2a94941f65 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8452,6 +8452,10 @@ class ProviderConfigManager: return litellm.InfinityEmbeddingConfig() elif litellm.LlmProviders.SAMBANOVA == provider: return litellm.SambaNovaEmbeddingConfig() + elif litellm.LlmProviders.OCI == provider: + from litellm.llms.oci.embed.transformation import OCIEmbedConfig + + return OCIEmbedConfig() elif ( litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider @@ -8509,10 +8513,6 @@ class ProviderConfigManager: return SagemakerEmbeddingConfig.get_model_config(model) elif litellm.LlmProviders.PERPLEXITY == provider: return litellm.PerplexityEmbeddingConfig() - elif litellm.LlmProviders.OCI == provider: - from litellm.llms.oci.embed.transformation import OCIEmbeddingConfig - - return OCIEmbeddingConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2140493ec4..514862516e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26018,6 +26018,32 @@ "supports_vision": true, "supports_web_search": true }, + "oci/meta.llama-3.1-8b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, "litellm_provider": "oci", @@ -26028,7 +26054,8 @@ "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-3.2-90b-vision-instruct": { "input_cost_per_token": 2e-06, @@ -26041,6 +26068,7 @@ "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, "supports_response_schema": false, + "supports_native_streaming": true, "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { @@ -26053,31 +26081,35 @@ "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 512000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true }, "oci/meta.llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 192000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 10485760, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3": { "input_cost_per_token": 3e-06, @@ -26089,7 +26121,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-fast": { "input_cost_per_token": 5e-06, @@ -26101,7 +26134,8 @@ "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini": { "input_cost_per_token": 3e-07, @@ -26113,7 +26147,8 @@ "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini-fast": { "input_cost_per_token": 6e-07, @@ -26125,7 +26160,8 @@ "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-4": { "input_cost_per_token": 3e-06, @@ -26137,7 +26173,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-latest": { "input_cost_per_token": 1.56e-06, @@ -26149,7 +26186,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-a-03-2025": { "input_cost_per_token": 1.56e-06, @@ -26161,7 +26199,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-plus-latest": { "input_cost_per_token": 1.56e-06, @@ -26173,7 +26212,86 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/cohere.command-a-vision": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true + }, + "oci/cohere.command-a-reasoning": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": false, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/cohere.embed-multilingual-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "mode": "embedding", + "output_vector_size": 1024, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { "input_cost_per_token": 1.56e-06, @@ -26249,18 +26367,6 @@ "supports_response_schema": false, "supports_vision": true }, - "oci/meta.llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "litellm_provider": "oci", - "max_input_tokens": 128000, - "max_output_tokens": 4000, - "max_tokens": 4000, - "mode": "chat", - "output_cost_per_token": 7.2e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": false - }, "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -26333,42 +26439,48 @@ "supports_function_calling": true, "supports_response_schema": false }, - "oci/google.gemini-2.5-pro": { + "oci/openai.gpt-5": { "input_cost_per_token": 1.25e-06, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-flash": { - "input_cost_per_token": 1.5e-07, + "oci/openai.gpt-5-mini": { + "input_cost_per_token": 2.5e-07, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 6e-07, + "output_cost_per_token": 2e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-flash-lite": { - "input_cost_per_token": 7.5e-08, + "oci/openai.gpt-5-nano": { + "input_cost_per_token": 5e-08, "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 3e-07, + "output_cost_per_token": 4e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, + "supports_native_streaming": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_vision": true }, diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 07af2735df..254d700ee5 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -47,6 +47,9 @@ IGNORE_FUNCTIONS = [ "_read_image_bytes", # max depth set. "_get_masked_values", # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts. "_redact_sensitive_litellm_params", # max depth set (default 10). + "_resolve", # OCI: $ref resolver bounded by `resolving_stack` cycle guard. + "resolve_oci_schema_anyof", # OCI: bounded by JSON-schema tree depth (no cycles possible in well-formed input). + "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. ] diff --git a/tests/integration/oci_proxy_test_config.yaml b/tests/integration/oci_proxy_test_config.yaml new file mode 100644 index 0000000000..bccb58d27f --- /dev/null +++ b/tests/integration/oci_proxy_test_config.yaml @@ -0,0 +1,24 @@ +model_list: + - model_name: oci-cohere-command + litellm_params: + model: oci/cohere.command-latest + - model_name: oci-llama + litellm_params: + model: oci/meta.llama-3.3-70b-instruct + - model_name: oci-gemini + litellm_params: + model: oci/google.gemini-2.5-flash + - model_name: oci-grok + litellm_params: + model: oci/xai.grok-3-mini + - model_name: oci-embed + litellm_params: + model: oci/cohere.embed-v4.0 + model_info: + mode: embedding + +general_settings: + master_key: sk-1234 + +litellm_settings: + drop_params: True diff --git a/tests/integration/test_oci_integration.py b/tests/integration/test_oci_integration.py new file mode 100644 index 0000000000..94b8930bce --- /dev/null +++ b/tests/integration/test_oci_integration.py @@ -0,0 +1,669 @@ +""" +OCI Generative AI — end-to-end integration tests. + +These tests make REAL calls to OCI. They are skipped automatically when the +standard ~/.oci/config is absent or when OCI_TEST_COMPARTMENT_ID is not set. + +Prerequisites +------------- +- ~/.oci/config with a valid [DEFAULT] profile +- Private key referenced by key_file in that profile +- Sufficient IAM policies to call the Generative AI inference service + +Environment variables (all optional — fall back to ~/.oci/config values): + OCI_TEST_REGION OCI region (default: us-chicago-1) + OCI_TEST_COMPARTMENT_ID compartment OCID (default: tenancy root from config) + +Run only these tests: + pytest tests/integration/test_oci_integration.py -v +""" + +import math +import os +import sys +from typing import NamedTuple, Optional + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + +OCI_CONFIG_FILE = os.path.expanduser("~/.oci/config") +_OCI_AVAILABLE = os.path.isfile(OCI_CONFIG_FILE) + +pytestmark = pytest.mark.skipif( + not _OCI_AVAILABLE, + reason="~/.oci/config not found — skipping OCI integration tests", +) + + +def _load_oci_config(): + """Load OCI config from the profile named by ``OCI_CONFIG_PROFILE`` env var, + falling back to ``[DEFAULT]``. Lets CI/local runs target a specific profile + without needing a ``[DEFAULT]`` section in ``~/.oci/config``.""" + oci = pytest.importorskip("oci") + profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT") + return oci.config.from_file(profile_name=profile) + + +@pytest.fixture(scope="module") +def oci_signer(): + """Return an oci.Signer (or SecurityTokenSigner for session-token profiles) + built from ~/.oci/config — profile chosen via OCI_CONFIG_PROFILE.""" + oci = pytest.importorskip("oci") + config = _load_oci_config() + + # Session-token profiles carry a `security_token_file` instead of a user + # OCID; build the corresponding signer in that case. + if "security_token_file" in config: + with open(os.path.expanduser(config["security_token_file"])) as f: + token = f.read().strip() + private_key = oci.signer.load_private_key_from_file( + config["key_file"], config.get("pass_phrase") + ) + return oci.auth.signers.SecurityTokenSigner(token, private_key) + + return oci.Signer( + tenancy=config["tenancy"], + user=config["user"], + fingerprint=config["fingerprint"], + private_key_file_location=config["key_file"], + ) + + +@pytest.fixture(scope="module") +def oci_params(oci_signer) -> dict: + """Common OCI call-time parameters shared by all tests.""" + config = _load_oci_config() + compartment_id = os.environ.get("OCI_TEST_COMPARTMENT_ID", config["tenancy"]) + region = os.environ.get("OCI_TEST_REGION", "us-chicago-1") + return { + "oci_signer": oci_signer, + "oci_compartment_id": compartment_id, + "oci_region": region, + } + + +# --------------------------------------------------------------------------- +# Model registry +# +# Each entry drives the runtime pivot inside OCI's own transformation layer — +# the tests themselves are format-agnostic. Per-model quirks are captured in +# the config fields below rather than in separate test classes. +# --------------------------------------------------------------------------- + + +class _M(NamedTuple): + """Per-model test configuration.""" + + model: str + max_tokens: int + # Reasoning models (Gemini 2.5, Grok mini) may return None content when the + # reasoning budget is exhausted before the answer token budget starts. + reasoning: bool = False + # tool_choice value to send; None means omit the parameter entirely. + tool_choice: Optional[str] = "auto" + # Whether to include the model in tool-use parametrize list. + supports_tool_use: bool = True + + +# All chat models under test. +CHAT_MODELS = [ + pytest.param(_M("meta.llama-3.3-70b-instruct", 64), id="meta"), + pytest.param(_M("google.gemini-2.5-flash", 200, reasoning=True), id="google"), + pytest.param(_M("xai.grok-3-mini", 100, reasoning=True), id="xai"), + pytest.param(_M("cohere.command-latest", 64, tool_choice=None), id="cohere"), +] + +# Subset of models that reliably support tool use in OCI. +# xAI Grok mini is omitted — OCI does not expose tool-use for it yet. +TOOL_USE_MODELS = [ + pytest.param(_M("meta.llama-3.3-70b-instruct", 100), id="meta"), + pytest.param(_M("cohere.command-latest", 200, tool_choice=None), id="cohere"), + pytest.param(_M("google.gemini-2.5-flash", 200, reasoning=True), id="google"), +] + +# Simple weather tool used by all tool-use tests. +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string", "description": "The city name."}}, + "required": ["city"], + }, + }, +} + + +# --------------------------------------------------------------------------- +# Sync chat tests — model list drives the pivot, not separate test classes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_basic_completion(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.prompt_tokens > 0 + if not m.reasoning: + assert resp.choices[0].message.content is not None + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_usage_populated(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is 2+2?"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.usage.prompt_tokens > 0 + assert resp.usage.total_tokens >= resp.usage.prompt_tokens + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_system_message(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Say hello."}, + ], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_streaming(m: _M, oci_params): + import litellm + + chunks = list( + litellm.completion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Count to 3."}], + max_tokens=m.max_tokens, + stream=True, + **oci_params, + ) + ) + assert len(chunks) > 0 + # Reasoning models may stream only reasoning tokens and return empty content. + if not m.reasoning: + content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) + assert len(content) > 0 + + +@pytest.mark.parametrize( + "model", + ["cohere.command-latest", "cohere.command-r-plus-08-2024"], +) +def test_cohere_streaming_no_doubling(model, oci_params): + """Regression: OCI Cohere's terminal SSE event re-sends the full assembled + response in `text` alongside a populated `chatHistory`. Emitting that text + as another delta would concatenate the whole response onto the + already-streamed output (e.g. "How can I help?How can I help?"). + + Reported by @gotsysdba on PR #25177. Fix: drop terminal text when + `chatHistory` is present in `handle_cohere_stream_chunk`. + """ + import litellm + + streamed = "".join( + (c.choices[0].delta.content or "") + for c in litellm.completion( + model=f"oci/{model}", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=64, + stream=True, + **oci_params, + ) + if c.choices + ).strip() + + assert streamed, "expected non-empty streamed content" + + # Compare against a non-streamed call. With the doubling bug the streamed + # assembly is ~2x the real response; without it the two are the same order + # of magnitude (the model is non-deterministic, so allow generous slack). + non_streamed = ( + litellm.completion( + model=f"oci/{model}", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=64, + **oci_params, + ) + .choices[0] + .message.content + or "" + ).strip() + + assert len(streamed) < 2 * len(non_streamed) + 10, ( + f"streamed output appears doubled — " + f"streamed={len(streamed)} chars vs non_streamed={len(non_streamed)} chars\n" + f"streamed: {streamed!r}\n" + f"non_streamed: {non_streamed!r}" + ) + + # Stronger signal: the very start of the response should not appear twice. + head = streamed[:12] + assert streamed.count(head) == 1, ( + f"streamed output contains its own prefix {head!r} more than once — " + f"likely the terminal chunk re-emitted the full response.\n" + f"streamed: {streamed!r}" + ) + + +@pytest.mark.parametrize("m", CHAT_MODELS) +def test_multi_turn(m: _M, oci_params): + import litellm + + resp = litellm.completion( + model=f"oci/{m.model}", + messages=[ + {"role": "user", "content": "My name is Alice."}, + {"role": "assistant", "content": "Nice to meet you, Alice!"}, + {"role": "user", "content": "What is my name?"}, + ], + max_tokens=m.max_tokens, + **oci_params, + ) + # Reasoning models may have None content; skip text assertion for them. + content = resp.choices[0].message.content or "" + if not m.reasoning: + assert "Alice" in content + + +# --------------------------------------------------------------------------- +# Async chat tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", CHAT_MODELS) +async def test_async_completion(m: _M, oci_params): + import litellm + + resp = await litellm.acompletion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=m.max_tokens, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.total_tokens > 0 + if not m.reasoning: + assert resp.choices[0].message.content is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", CHAT_MODELS) +async def test_async_streaming(m: _M, oci_params): + import litellm + + chunks = [] + async for chunk in await litellm.acompletion( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "Count to 3."}], + max_tokens=m.max_tokens, + stream=True, + **oci_params, + ): + chunks.append(chunk) + + assert len(chunks) > 0 + if not m.reasoning: + content = "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) + assert len(content) > 0 + + +# --------------------------------------------------------------------------- +# Tool-use tests +# --------------------------------------------------------------------------- + + +def _assert_tool_call(resp, expected_tool: str = "get_weather"): + """Assert the response contains the expected tool call (or a plain stop).""" + choice = resp.choices[0] + assert choice.finish_reason in ("tool_calls", "stop") + if choice.finish_reason == "tool_calls": + assert choice.message.tool_calls is not None + assert len(choice.message.tool_calls) > 0 + assert choice.message.tool_calls[0].function.name == expected_tool + + +@pytest.mark.parametrize("m", TOOL_USE_MODELS) +def test_tool_use(m: _M, oci_params): + import litellm + + call_kwargs = dict( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + max_tokens=m.max_tokens, + **oci_params, + ) + if m.tool_choice is not None: + call_kwargs["tool_choice"] = m.tool_choice + + resp = litellm.completion(**call_kwargs) + _assert_tool_call(resp) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("m", TOOL_USE_MODELS) +async def test_async_tool_use(m: _M, oci_params): + import litellm + + call_kwargs = dict( + model=f"oci/{m.model}", + messages=[{"role": "user", "content": "What is the weather in Berlin?"}], + tools=[_WEATHER_TOOL], + max_tokens=m.max_tokens, + **oci_params, + ) + if m.tool_choice is not None: + call_kwargs["tool_choice"] = m.tool_choice + + resp = await litellm.acompletion(**call_kwargs) + _assert_tool_call(resp) + + +# --------------------------------------------------------------------------- +# Reasoning-effort tests (reasoning models only) +# --------------------------------------------------------------------------- + +# Reasoning model that accepts the `reasoningEffort` parameter on OCI. +# Not every reasoning model does — xai.grok-4-fast-reasoning, for example, +# rejects it with a 400. +_REASONING_MODEL = "xai.grok-3-mini" + + +@pytest.mark.parametrize("effort", ["low", "medium", "high"]) +def test_reasoning_effort_lowercase_accepted(effort, oci_params): + """OpenAI clients send lowercase reasoning_effort; OCI requires uppercase. + The transform layer should uppercase it transparently.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort=effort, + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + assert resp.usage.prompt_tokens > 0 + + +def test_reasoning_effort_disable_mapped_to_none(oci_params): + """OpenAI's 'disable' maps to OCI's 'NONE'. Without this mapping the + request 400s.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="disable", + **oci_params, + ) + assert resp.choices[0].finish_reason is not None + + +def test_reasoning_tokens_in_usage(oci_params): + """OCI returns completionTokensDetails.reasoningTokens on reasoning models; + LiteLLM should surface it on Usage.completion_tokens_details.""" + import litellm + + resp = litellm.completion( + model=f"oci/{_REASONING_MODEL}", + messages=[{"role": "user", "content": "What is 2+2? One word."}], + max_tokens=200, + reasoning_effort="low", + **oci_params, + ) + assert resp.usage.completion_tokens_details is not None + assert resp.usage.completion_tokens_details.reasoning_tokens is not None + assert resp.usage.completion_tokens_details.reasoning_tokens > 0 + + +# --------------------------------------------------------------------------- +# Embedding tests +# --------------------------------------------------------------------------- + + +class TestOCIEmbeddings: + + def test_english_v3_basic(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1024 + assert resp.usage.prompt_tokens > 0 + + def test_english_v3_batch(self, oci_params): + import litellm + + texts = [ + "The quick brown fox", + "jumps over the lazy dog", + "Paris is the capital of France", + ] + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=texts, + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 3 + for i, item in enumerate(resp.data): + assert item["index"] == i + assert len(item["embedding"]) == 1024 + + def test_multilingual_v3(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-multilingual-v3.0", + input=["Bonjour le monde", "Hola mundo"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 2 + assert len(resp.data[0]["embedding"]) == 1024 + + def test_search_query_input_type(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["What is the capital of France?"], + input_type="SEARCH_QUERY", + **oci_params, + ) + assert len(resp.data[0]["embedding"]) == 1024 + + def test_semantic_similarity(self, oci_params): + """Semantically similar texts should have higher cosine similarity.""" + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=[ + "The cat sat on the mat", + "A feline rested on the rug", + "The stock market crashed today", + ], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + + def cosine(a, b): + dot = sum(x * y for x, y in zip(a, b)) + mag_a = math.sqrt(sum(x**2 for x in a)) + mag_b = math.sqrt(sum(x**2 for x in b)) + return dot / (mag_a * mag_b) + + cat1 = resp.data[0]["embedding"] + cat2 = resp.data[1]["embedding"] + stock = resp.data[2]["embedding"] + sim_cats = cosine(cat1, cat2) + sim_diff = cosine(cat1, stock) + assert ( + sim_cats > sim_diff + ), f"Expected similar sentences to score higher ({sim_cats:.3f} vs {sim_diff:.3f})" + + def test_embed_v4(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-v4.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1536 + + def test_usage_tokens(self, oci_params): + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["short text", "another short text"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert resp.usage.prompt_tokens > 0 + assert resp.usage.total_tokens == resp.usage.prompt_tokens + + +# --------------------------------------------------------------------------- +# Async embedding tests +# --------------------------------------------------------------------------- + + +class TestOCIAsyncEmbeddings: + + @pytest.mark.asyncio + async def test_async_embedding_basic(self, oci_params): + import litellm + + resp = await litellm.aembedding( + model="oci/cohere.embed-english-v3.0", + input=["Hello world"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 1 + assert len(resp.data[0]["embedding"]) == 1024 + assert resp.usage.prompt_tokens > 0 + + @pytest.mark.asyncio + async def test_async_embedding_batch(self, oci_params): + import litellm + + texts = ["The quick brown fox", "jumps over the lazy dog"] + resp = await litellm.aembedding( + model="oci/cohere.embed-english-v3.0", + input=texts, + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data) == 2 + assert all(len(item["embedding"]) == 1024 for item in resp.data) + + @pytest.mark.asyncio + async def test_async_embedding_multilingual(self, oci_params): + import litellm + + resp = await litellm.aembedding( + model="oci/cohere.embed-multilingual-v3.0", + input=["Bonjour le monde"], + input_type="SEARCH_DOCUMENT", + **oci_params, + ) + assert len(resp.data[0]["embedding"]) == 1024 + + +# --------------------------------------------------------------------------- +# Env-var credential path +# --------------------------------------------------------------------------- + + +class TestOCIEnvVarCredentials: + """Verify that OCI_* env vars are picked up without explicit params.""" + + def test_completion_via_env_vars(self, monkeypatch): + """Completion works when credentials are set through environment variables.""" + pytest.importorskip("oci") + config = _load_oci_config() + key_path = os.path.expanduser(config["key_file"]) + + with open(key_path) as f: + key_pem = f.read() + + monkeypatch.setenv("OCI_REGION", "us-chicago-1") + monkeypatch.setenv("OCI_USER", config["user"]) + monkeypatch.setenv("OCI_FINGERPRINT", config["fingerprint"]) + monkeypatch.setenv("OCI_TENANCY", config["tenancy"]) + monkeypatch.setenv("OCI_KEY", key_pem) + monkeypatch.setenv("OCI_COMPARTMENT_ID", config["tenancy"]) + + import litellm + + resp = litellm.completion( + model="oci/meta.llama-3.3-70b-instruct", + messages=[{"role": "user", "content": "Reply with only the word: pong"}], + max_tokens=10, + ) + assert resp.choices[0].message.content is not None + + def test_embedding_via_env_vars(self, monkeypatch): + pytest.importorskip("oci") + config = _load_oci_config() + key_path = os.path.expanduser(config["key_file"]) + + with open(key_path) as f: + key_pem = f.read() + + monkeypatch.setenv("OCI_REGION", "us-chicago-1") + monkeypatch.setenv("OCI_USER", config["user"]) + monkeypatch.setenv("OCI_FINGERPRINT", config["fingerprint"]) + monkeypatch.setenv("OCI_TENANCY", config["tenancy"]) + monkeypatch.setenv("OCI_KEY", key_pem) + monkeypatch.setenv("OCI_COMPARTMENT_ID", config["tenancy"]) + + import litellm + + resp = litellm.embedding( + model="oci/cohere.embed-english-v3.0", + input=["hello"], + input_type="SEARCH_DOCUMENT", + ) + assert len(resp.data[0]["embedding"]) == 1024 diff --git a/tests/integration/test_oci_proxy_integration.py b/tests/integration/test_oci_proxy_integration.py new file mode 100644 index 0000000000..8bfcdd9048 --- /dev/null +++ b/tests/integration/test_oci_proxy_integration.py @@ -0,0 +1,274 @@ +""" +OCI GenAI — end-to-end **proxy** integration tests. + +Spins up the LiteLLM proxy (`litellm --config oci_proxy_test_config.yaml`) as a +subprocess, then sends OpenAI-shaped HTTP requests at it for the OCI models +declared in the test config: + + - oci-cohere-command (oci/cohere.command-latest) + - oci-llama (oci/meta.llama-3.3-70b-instruct) + - oci-gemini (oci/google.gemini-2.5-flash) + - oci-grok (oci/xai.grok-3-mini) + - oci-embed (oci/cohere.embed-v4.0) + +Skipped unless: + - ~/.oci/config exists + - The `oci` SDK is installed (handled by ``pytest.importorskip``) + +Environment variables honoured (passed through to the proxy subprocess): + OCI_CONFIG_PROFILE profile inside ~/.oci/config (default: DEFAULT) + OCI_REGION overrides region from the profile (default: us-chicago-1) + +Run with:: + + OCI_CONFIG_PROFILE=LUIGI_FRA_API OCI_REGION=us-chicago-1 \ + uv run pytest tests/integration/test_oci_proxy_integration.py -v -s + +The tests open a real socket on a free TCP port — no port collision with a +locally-running proxy. +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import time +from pathlib import Path +from typing import Iterator + +import httpx +import pytest + + +# --------------------------------------------------------------------------- +# Skip gate +# --------------------------------------------------------------------------- +OCI_CONFIG_FILE = os.path.expanduser("~/.oci/config") +pytestmark = pytest.mark.skipif( + not os.path.isfile(OCI_CONFIG_FILE), + reason="~/.oci/config not found — skipping OCI proxy integration tests", +) + + +CONFIG_PATH = Path(__file__).parent / "oci_proxy_test_config.yaml" +MASTER_KEY = "sk-1234" +STARTUP_TIMEOUT_S = 90.0 +REQUEST_TIMEOUT_S = 120.0 + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_for_health(base_url: str, proc: subprocess.Popen, deadline: float) -> None: + """Poll /health/liveliness until the proxy answers or the deadline expires.""" + while time.monotonic() < deadline: + if proc.poll() is not None: + output = proc.stdout.read() if proc.stdout else "" + raise RuntimeError( + f"litellm proxy exited early with code {proc.returncode}\n--- proxy output ---\n{output}" + ) + try: + r = httpx.get(f"{base_url}/health/liveliness", timeout=2.0) + if r.status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.5) + raise RuntimeError(f"litellm proxy did not become ready within {STARTUP_TIMEOUT_S}s") + + +def _oci_env_from_profile() -> dict[str, str]: + """Translate the active OCI profile into the OCI_* env vars the litellm + OCI provider expects. Only API-key profiles are supported; session-token + profiles would need an in-process signer and so are skipped here. + """ + oci = pytest.importorskip("oci") + profile = os.environ.get("OCI_CONFIG_PROFILE", "DEFAULT") + cfg = oci.config.from_file(profile_name=profile) + if "security_token_file" in cfg: + pytest.skip( + f"OCI profile {profile!r} uses session-token auth; " + "litellm's OCI provider needs an API-key profile for env-driven config" + ) + region = os.environ.get("OCI_REGION") or cfg.get("region") or "us-chicago-1" + return { + "OCI_USER": cfg["user"], + "OCI_FINGERPRINT": cfg["fingerprint"], + "OCI_TENANCY": cfg["tenancy"], + "OCI_COMPARTMENT_ID": os.environ.get("OCI_COMPARTMENT_ID", cfg["tenancy"]), + "OCI_KEY_FILE": os.path.expanduser(cfg["key_file"]), + "OCI_REGION": region, + } + + +@pytest.fixture(scope="module") +def proxy_url() -> Iterator[str]: + oci_env = _oci_env_from_profile() + + port = _free_port() + base_url = f"http://127.0.0.1:{port}" + + env = os.environ.copy() + env.update(oci_env) + # Avoid pulling in DB-backed features for this lightweight smoke run. + env.pop("DATABASE_URL", None) + env["STORE_MODEL_IN_DB"] = "False" + + # Prefer the `litellm` console script that lives next to the active + # Python so we inherit the test virtualenv. Fall back to PATH. + cli = Path(sys.executable).parent / "litellm" + if not cli.exists(): + cli = "litellm" + cmd = [ + str(cli), + "--config", + str(CONFIG_PATH), + "--port", + str(port), + "--host", + "127.0.0.1", + "--num_workers", + "1", + ] + + proc = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + _wait_for_health(base_url, proc, time.monotonic() + STARTUP_TIMEOUT_S) + yield base_url + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _auth_headers() -> dict: + return { + "Authorization": f"Bearer {MASTER_KEY}", + "Content-Type": "application/json", + } + + +def _chat_payload(model: str, *, stream: bool = False) -> dict: + return { + "model": model, + "messages": [ + {"role": "user", "content": "Reply with only the single word: pong"} + ], + "max_tokens": 64, + "stream": stream, + } + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +CHAT_MODELS = ["oci-cohere-command", "oci-llama", "oci-gemini", "oci-grok"] + + +@pytest.mark.parametrize("model", CHAT_MODELS) +def test_chat_completion_via_proxy(proxy_url: str, model: str) -> None: + """Non-streaming chat completion returns a well-formed OpenAI response.""" + r = httpx.post( + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json=_chat_payload(model), + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"{model} -> {r.status_code}: {r.text}" + body = r.json() + assert body["object"] == "chat.completion" + assert body["model"] == model + choices = body["choices"] + assert len(choices) >= 1 + msg = choices[0]["message"] + assert msg["role"] == "assistant" + # Reasoning models may return empty content if their budget covers only + # the thinking turn — accept either text or a non-empty reasoning field. + has_content = bool(msg.get("content")) + has_reasoning = bool(msg.get("reasoning_content")) or bool( + msg.get("reasoning") + ) + assert has_content or has_reasoning, f"empty assistant message for {model}: {msg}" + usage = body.get("usage") or {} + assert usage.get("total_tokens", 0) > 0 + + +@pytest.mark.parametrize("model", CHAT_MODELS) +def test_chat_completion_streaming_via_proxy(proxy_url: str, model: str) -> None: + """Streaming chat completion yields at least one data: chunk and a [DONE].""" + saw_chunk = False + saw_done = False + with httpx.stream( + "POST", + f"{proxy_url}/v1/chat/completions", + headers=_auth_headers(), + json=_chat_payload(model, stream=True), + timeout=REQUEST_TIMEOUT_S, + ) as r: + assert r.status_code == 200, f"{model} stream -> {r.status_code}: {r.read()!r}" + for line in r.iter_lines(): + if not line: + continue + if not line.startswith("data:"): + continue + payload = line[len("data:"):].strip() + if payload == "[DONE]": + saw_done = True + break + saw_chunk = True + assert saw_chunk, f"no streamed chunks for {model}" + assert saw_done, f"no [DONE] sentinel for {model}" + + +def test_embedding_via_proxy(proxy_url: str) -> None: + """OCI Cohere embedding endpoint returns a non-empty vector via the proxy.""" + r = httpx.post( + f"{proxy_url}/v1/embeddings", + headers=_auth_headers(), + json={"model": "oci-embed", "input": ["hello from the litellm proxy"]}, + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, f"embed -> {r.status_code}: {r.text}" + body = r.json() + assert body["object"] == "list" + assert body["model"] == "oci-embed" + data = body["data"] + assert len(data) == 1 + embedding = data[0]["embedding"] + assert isinstance(embedding, list) + assert len(embedding) >= 64 + assert all(isinstance(x, (int, float)) for x in embedding) + + +def test_model_list_advertises_oci_models(proxy_url: str) -> None: + """The /v1/models registry advertises every OCI alias from the config.""" + r = httpx.get( + f"{proxy_url}/v1/models", + headers=_auth_headers(), + timeout=REQUEST_TIMEOUT_S, + ) + assert r.status_code == 200, r.text + advertised = {row["id"] for row in r.json()["data"]} + for expected in CHAT_MODELS + ["oci-embed"]: + assert expected in advertised, f"{expected} missing from /v1/models: {advertised}" diff --git a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py index f96228a4cc..e9b3f82d1a 100644 --- a/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -14,7 +14,7 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.llms.oci.chat.transformation import OCIChatConfig -from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.common_utils import OCIError, sign_with_manual_credentials @pytest.fixture @@ -41,7 +41,7 @@ class TestOCIKeyNormalization: # We can't fully test signing without a real key, but we can verify # the error message indicates the key was processed (not a type error) with pytest.raises(Exception) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -67,7 +67,7 @@ class TestOCIKeyNormalization: } with pytest.raises(Exception) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -88,7 +88,7 @@ class TestOCIKeyNormalization: } with pytest.raises(OCIError) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, @@ -110,7 +110,7 @@ class TestOCIKeyNormalization: } with pytest.raises(OCIError) as exc_info: - config._sign_with_manual_credentials( + sign_with_manual_credentials( headers={}, optional_params=optional_params, request_data={"test": "data"}, diff --git a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py index 2a5cc6b8e3..e0911e1ef3 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_chat_transformation.py @@ -443,7 +443,7 @@ class TestOCIChatConfig: {"type": "TEXT", "text": "I am doing well, thank you!"} ], }, - "finishReason": "STOP", + "finishReason": "COMPLETE", } ], "timeCreated": created_time, @@ -490,9 +490,10 @@ class TestOCIChatConfig: assert result.usage.prompt_tokens == 10 # type: ignore assert result.usage.completion_tokens == 20 # type: ignore assert result.usage.total_tokens == 30 # type: ignore - # These are not handled in the transformer, TBH no idea why they are here - # but, for now, they seem to be always None - assert result.usage.completion_tokens_details is None + # reasoningTokens from OCI's completionTokensDetails is surfaced on + # Usage.completion_tokens_details.reasoning_tokens. + assert result.usage.completion_tokens_details is not None + assert result.usage.completion_tokens_details.reasoning_tokens == 20 assert result.usage.prompt_tokens_details is None def test_transform_response_with_tool_calls(self): @@ -765,3 +766,557 @@ class TestOCISignerSupport: ) assert wrapper.path_url == "/api/v1/chat" + + +class TestOCISplitChunks: + """ + Unit tests for the SSE split_chunks helpers used in sync and async streaming. + + These validate the fix for: + - Sync: JSONDecodeError when iter_text() returns chunks spanning multiple events + - Async: whitespace-only chunks being yielded before stripping (Greptile P2) + """ + + def _run_sync_split(self, raw_chunks): + """Invoke the sync split_chunks logic directly (extracted for testability).""" + results = [] + for item in raw_chunks: + for chunk in item.split("\n\n"): + stripped = chunk.strip() + if stripped: + results.append(stripped) + return results + + async def _run_async_split(self, raw_chunks): + """Invoke the async split_chunks logic directly.""" + results = [] + + async def _gen(): + for c in raw_chunks: + yield c + + async for item in _gen(): + for chunk in item.split("\n\n"): + stripped = chunk.strip() + if stripped: + results.append(stripped) + return results + + def test_sync_single_event_per_chunk(self): + """Normal case: one SSE event per iter_text() chunk.""" + chunks = ['data: {"text":"hello"}', 'data: {"text":"world"}'] + assert self._run_sync_split(chunks) == [ + 'data: {"text":"hello"}', + 'data: {"text":"world"}', + ] + + def test_sync_multiple_events_in_one_chunk(self): + """iter_text() returns two SSE events concatenated — must be split.""" + chunks = ['data: {"text":"a"}\n\ndata: {"text":"b"}'] + assert self._run_sync_split(chunks) == [ + 'data: {"text":"a"}', + 'data: {"text":"b"}', + ] + + def test_sync_whitespace_only_chunks_discarded(self): + """Whitespace between events must not be yielded.""" + chunks = ["data: {}\n\n \n\ndata: {}"] + result = self._run_sync_split(chunks) + assert result == ["data: {}", "data: {}"] + + def test_sync_empty_string_discarded(self): + """Empty string produced by splitting trailing \\n\\n must be discarded.""" + chunks = ["data: {}\n\n"] + assert self._run_sync_split(chunks) == ["data: {}"] + + @pytest.mark.asyncio + async def test_async_whitespace_only_chunks_discarded(self): + """ + Regression test for Greptile P2: async version was checking `if not chunk` + BEFORE stripping, so '\\n ' would pass the guard and yield '' downstream, + causing ValueError in chunk_creator ('Chunk does not start with data:'). + """ + chunks = ["data: {}\n\n \n\ndata: {}"] + result = await self._run_async_split(chunks) + assert result == ["data: {}", "data: {}"] + + @pytest.mark.asyncio + async def test_async_empty_string_discarded(self): + """Trailing \\n\\n must not produce an empty yielded chunk in async path.""" + chunks = ["data: {}\n\n"] + result = await self._run_async_split(chunks) + assert result == ["data: {}"] + + @pytest.mark.asyncio + async def test_async_multiple_events_in_one_chunk(self): + """Async path must split concatenated SSE events just like sync.""" + chunks = ['data: {"text":"x"}\n\ndata: {"text":"y"}'] + result = await self._run_async_split(chunks) + assert result == ['data: {"text":"x"}', 'data: {"text":"y"}'] + + +class TestOCIProviderEmbeddingConfig: + """ + Verifies that get_provider_embedding_config returns OCIEmbedConfig for OCI + and that the dead duplicate elif branch has been removed (Greptile P1). + """ + + def test_returns_oci_embed_config(self): + from litellm.llms.oci.embed.transformation import OCIEmbedConfig + from litellm.utils import ProviderConfigManager + from litellm.types.utils import LlmProviders + + config = ProviderConfigManager.get_provider_embedding_config( + model="cohere.embed-english-v3.0", + provider=LlmProviders.OCI, + ) + assert isinstance(config, OCIEmbedConfig) + + def test_no_duplicate_oci_branch(self): + """ + Ensure utils.py does not contain two separate OCI embedding branches. + The dead code was removed in commit 64dfbe2b; this test guards against + regression (e.g. a future merge re-introducing it). + """ + import inspect + from litellm.utils import ProviderConfigManager + + source = inspect.getsource(ProviderConfigManager.get_provider_embedding_config) + oci_count = source.count("LlmProviders.OCI") + assert oci_count == 1, ( + f"Expected exactly 1 OCI branch in get_provider_embedding_config, found {oci_count}. " + "A duplicate dead-code branch may have been reintroduced." + ) + + +class TestOCICohereParamMapping: + """ + Unit tests for Bug 3 (stop → stopSequences) and Bug 4 (hardcoded defaults removed). + """ + + def _make_config(self): + return OCIChatConfig() + + def test_cohere_stop_maps_to_stop_sequences(self): + """Bug 3: Cohere API uses 'stopSequences', not 'stop'.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"stop": ["END", "STOP"]}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + assert "stopSequences" in result, "stop should map to stopSequences for Cohere" + assert result["stopSequences"] == ["END", "STOP"] + assert "stop" not in result + + def test_generic_stop_maps_to_stop(self): + """GENERIC vendors (Meta, Google, xAI) keep 'stop' as-is.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"stop": ["END"]}, + optional_params={}, + model="meta.llama-3.3-70b-instruct", + drop_params=False, + ) + assert result.get("stop") == ["END"] + assert "stopSequences" not in result + + def test_cohere_no_hardcoded_defaults(self): + """Bug 4: Cohere calls must not inject maxTokens/temperature/topK/topP/frequencyPenalty + when the user hasn't provided them.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + for injected in ( + "maxTokens", + "temperature", + "topK", + "topP", + "frequencyPenalty", + ): + assert ( + injected not in result + ), f"'{injected}' should not be injected when user did not provide it" + + def test_cohere_explicit_params_still_passed(self): + """User-provided Cohere params must still be forwarded correctly.""" + config = self._make_config() + result = config.map_openai_params( + non_default_params={"max_tokens": 200, "temperature": 0.5}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + assert result.get("maxTokens") == 200 + assert result.get("temperature") == 0.5 + + +class TestOCIReasoningEffort: + """ + Reasoning-effort handling for GENERIC reasoning models: + - OpenAI clients send lowercase ("low"/"medium"/"high"); OCI requires uppercase. + - OpenAI's "disable" maps to OCI's "NONE". + - Cohere on OCI has no reasoning models — the param is unsupported there. + """ + + def _build_chat_request(self, model: str, optional_params: dict) -> dict: + """Drive optional params through map → _get_optional_params and read + the resulting chatRequest body via transform_request.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + mapped = config.map_openai_params( + non_default_params=optional_params, + optional_params={}, + model=model, + drop_params=False, + ) + body = config.transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={**BASE_OCI_PARAMS, **mapped}, + litellm_params={}, + headers={}, + ) + return body["chatRequest"] + + def test_reasoning_effort_lowercase_uppercased(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "low"}, + ) + assert chat_request.get("reasoningEffort") == "LOW" + + def test_reasoning_effort_disable_mapped_to_none(self): + chat_request = self._build_chat_request( + "xai.grok-4-fast-reasoning", + {"reasoning_effort": "disable"}, + ) + assert chat_request.get("reasoningEffort") == "NONE" + + def test_reasoning_effort_already_uppercase_preserved(self): + chat_request = self._build_chat_request( + "openai.gpt-5", + {"reasoning_effort": "HIGH"}, + ) + assert chat_request.get("reasoningEffort") == "HIGH" + + def test_reasoning_effort_unsupported_on_cohere_dropped(self): + """drop_params=True → silently drop reasoning_effort for Cohere.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + assert "reasoning_effort" not in result + assert "reasoningEffort" not in result + + def test_reasoning_effort_unsupported_on_cohere_raises(self): + """drop_params=False → raise rather than ship a payload Cohere will reject.""" + from litellm.llms.oci.chat.transformation import OCIChatConfig + from litellm.llms.oci.common_utils import OCIError + + config = OCIChatConfig() + with pytest.raises(OCIError): + config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=False, + ) + + def test_reasoning_tokens_extracted_from_usage(self): + """OCI's completionTokensDetails.reasoningTokens flows into + Usage.completion_tokens_details.reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4-fast-reasoning", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + "completionTokensDetails": {"reasoningTokens": 7}, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4-fast-reasoning", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 7 + + def test_reasoning_tokens_absent_when_no_details(self): + """When OCI omits completionTokensDetails, Usage has no reasoning_tokens.""" + from litellm.llms.oci.chat.generic import handle_generic_response + + created_time = ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + oci_response = { + "modelId": "xai.grok-4", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "GENERIC", + "choices": [ + { + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "ok"}], + }, + "finishReason": "STOP", + } + ], + "timeCreated": created_time, + "usage": { + "promptTokens": 5, + "completionTokens": 12, + "totalTokens": 17, + }, + }, + } + raw = httpx.Response(status_code=200, json=oci_response) + result = handle_generic_response( + json_data=oci_response, + model="xai.grok-4", + model_response=ModelResponse(), + raw_response=raw, + ) + usage = result.usage # type: ignore[attr-defined] + assert usage.completion_tokens_details is None + + +class TestOCIStreamingSignedBody: + """ + Unit test for Bug 1: sync and async streaming paths must use signed_json_body + when provided, not re-serialize data with json.dumps(). + """ + + def test_get_custom_stream_wrapper_uses_signed_body(self, monkeypatch): + """ + When signed_json_body is provided, the POST must use that exact bytes object, + not json.dumps(data) — otherwise the RSA-SHA256 signature is invalid. + """ + import httpx + from unittest.mock import MagicMock, patch + + config = OCIChatConfig() + signed_bytes = b'{"signed": true}' + posted_data = {} + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter([]) + + mock_client = MagicMock() + mock_client.post.return_value = mock_response + + def capture_post(url, **kwargs): + posted_data["data"] = kwargs.get("data") + return mock_response + + mock_client.post.side_effect = capture_post + + mock_logging = MagicMock() + + config.get_sync_custom_stream_wrapper( + api_base="https://example.com", + headers={}, + data={"key": "value"}, + messages=[], + model="meta.llama-3.3-70b-instruct", + custom_llm_provider="oci", + logging_obj=mock_logging, + client=mock_client, + signed_json_body=signed_bytes, + ) + + assert ( + posted_data["data"] == signed_bytes + ), "Streaming must use signed_json_body, not re-serialize data" + + def test_get_custom_stream_wrapper_fallback_without_signed_body(self, monkeypatch): + """When signed_json_body is None, fall back to json.dumps(data).""" + import json + from unittest.mock import MagicMock + + config = OCIChatConfig() + posted_data = {} + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter([]) + + mock_client = MagicMock() + + def capture_post(url, **kwargs): + posted_data["data"] = kwargs.get("data") + return mock_response + + mock_client.post.side_effect = capture_post + + mock_logging = MagicMock() + payload = {"key": "value"} + + config.get_sync_custom_stream_wrapper( + api_base="https://example.com", + headers={}, + data=payload, + messages=[], + model="meta.llama-3.3-70b-instruct", + custom_llm_provider="oci", + logging_obj=mock_logging, + client=mock_client, + signed_json_body=None, + ) + + assert posted_data["data"] == json.dumps( + payload + ), "Without signed_json_body, must fall back to json.dumps(data)" + + +# --------------------------------------------------------------------------- +# Additional coverage: error paths in validate_environment, transform_request, +# transform_response, and map_openai_params +# --------------------------------------------------------------------------- + + +class TestOCIChatConfigErrorPaths: + def test_validate_environment_empty_messages_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="messages"): + config.validate_environment( + headers={}, + model=TEST_MODEL_NAME, + messages=[], + optional_params={ + "oci_signer": MagicMock(), + "oci_compartment_id": TEST_COMPARTMENT_ID, + }, + litellm_params={}, + ) + + def test_transform_request_missing_compartment_id_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="oci_compartment_id"): + config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params={}, + litellm_params={}, + headers={}, + ) + + def test_transform_request_cohere_no_user_message_raises(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="user message"): + config.transform_request( + model="cohere.command-latest", + messages=[{"role": "system", "content": "You are helpful."}], # type: ignore + optional_params={"oci_compartment_id": TEST_COMPARTMENT_ID}, + litellm_params={}, + headers={}, + ) + + def test_transform_response_error_key_raises(self): + config = OCIChatConfig() + response = httpx.Response( + status_code=400, + json={"error": "model not found"}, + ) + with pytest.raises(Exception, match="model not found"): + config.transform_response( + model=TEST_MODEL_NAME, + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + def test_map_openai_params_unsupported_param_raises_without_drop(self): + config = OCIChatConfig() + with pytest.raises(Exception, match="not supported on OCI"): + config.map_openai_params( + non_default_params={"audio": {"voice": "alloy"}}, + optional_params={}, + model=TEST_MODEL_NAME, + drop_params=False, + ) + + def test_map_openai_params_unsupported_param_dropped(self): + config = OCIChatConfig() + result = config.map_openai_params( + non_default_params={"audio": {"voice": "alloy"}}, + optional_params={}, + model=TEST_MODEL_NAME, + drop_params=True, + ) + assert "audio" not in result + + def test_transform_request_tool_choice_string_mapped(self): + config = OCIChatConfig() + result = config.transform_request( + model=TEST_MODEL_NAME, + messages=TEST_MESSAGES, # type: ignore + optional_params={ + "oci_compartment_id": TEST_COMPARTMENT_ID, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "function": { + "name": "fn", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + }, + litellm_params={}, + headers={}, + ) + assert result["chatRequest"]["toolChoice"] == {"type": "AUTO"} + + +import pytest +from unittest.mock import MagicMock diff --git a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py index 388cb6224f..cc914a22ee 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_cohere_tool_calls.py @@ -5,10 +5,14 @@ import json from unittest.mock import patch, MagicMock from litellm import ModelResponse +from litellm.llms.oci.chat.cohere import ( + adapt_messages_to_cohere_standard, + adapt_tool_definitions_to_cohere_standard, +) from litellm.llms.oci.chat.transformation import ( OCIChatConfig, - get_vendor_from_model, OCIStreamWrapper, + get_vendor_from_model, ) from litellm.types.llms.oci import OCIVendors @@ -75,7 +79,7 @@ class TestOCICohereToolCalls: ] # Transform tools - cohere_tools = config.adapt_tool_definitions_to_cohere_standard(openai_tools) + cohere_tools = adapt_tool_definitions_to_cohere_standard(openai_tools) # Verify transformation assert len(cohere_tools) == 2 @@ -90,13 +94,16 @@ class TestOCICohereToolCalls: # Check location parameter location_param = weather_tool.parameterDefinitions["location"] assert location_param.description == "The city or location to get weather for" - assert location_param.type == "string" + assert location_param.type == "str" assert location_param.isRequired == True # Check unit parameter unit_param = weather_tool.parameterDefinitions["unit"] - assert unit_param.description == "Temperature unit (celsius or fahrenheit)" - assert unit_param.type == "string" + assert ( + unit_param.description + == "Temperature unit (celsius or fahrenheit). Allowed values: ['celsius', 'fahrenheit']" + ) + assert unit_param.type == "str" assert unit_param.isRequired == False # Check second tool @@ -107,7 +114,7 @@ class TestOCICohereToolCalls: expression_param = calc_tool.parameterDefinitions["expression"] assert expression_param.description == "Mathematical expression to evaluate" - assert expression_param.type == "string" + assert expression_param.type == "str" assert expression_param.isRequired == True def test_cohere_request_with_tools(self): @@ -157,13 +164,6 @@ class TestOCICohereToolCalls: assert chat_request["message"] == "What's the weather like in Tokyo?" assert chat_request["chatHistory"] == [] - # Verify default parameters are included - assert chat_request["maxTokens"] == 600 - assert chat_request["temperature"] == 1 - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 - # Verify tools are transformed correctly assert "tools" in chat_request assert len(chat_request["tools"]) == 1 @@ -226,7 +226,7 @@ class TestOCICohereToolCalls: assert len(result.choices[0].message.tool_calls) == 1 tool_call = result.choices[0].message.tool_calls[0] - assert tool_call.id == "call_0" + assert tool_call.id.startswith("call_") assert tool_call.type == "function" assert tool_call.function.name == "get_weather" assert tool_call.function.arguments == '{"location": "Tokyo"}' @@ -324,22 +324,26 @@ class TestOCICohereToolCalls: }, ] - chat_history = config.adapt_messages_to_cohere_standard(messages) + chat_history = adapt_messages_to_cohere_standard(messages) - # First message is the user message - assert chat_history[0].role == "USER" - assert chat_history[0].message == "What's the weather?" + # The last user message is consumed by the request's top-level `message` + # field, so chatHistory carries the assistant tool call and tool result. + assert len(chat_history) == 2 - # Second message is the assistant with tool calls and no text - assistant_msg = chat_history[1] + assistant_msg = chat_history[0] assert assistant_msg.role == "CHATBOT" assert assistant_msg.message is None or assistant_msg.message == "" assert assistant_msg.toolCalls is not None assert len(assistant_msg.toolCalls) == 1 assert assistant_msg.toolCalls[0].name == "get_weather" + tool_msg = chat_history[1] + assert tool_msg.role == "TOOL" + assert tool_msg.toolResults[0].call.name == "get_weather" + assert tool_msg.toolResults[0].outputs[0]["output"] == "Sunny, 25C" + def test_cohere_chat_history_with_tool_calls(self): - """Test chat history transformation with tool calls""" + """Tool results trailing the last user turn must be preserved in chatHistory.""" config = OCIChatConfig() messages = [ @@ -365,28 +369,29 @@ class TestOCICohereToolCalls: }, ] - chat_history = config.adapt_messages_to_cohere_standard(messages) + chat_history = adapt_messages_to_cohere_standard(messages) - # Verify chat history structure (excludes last message) + # The last user message becomes the request's top-level `message`. + # Everything else — including the trailing tool result — must remain in + # chatHistory so the model can see the tool output. assert len(chat_history) == 2 - # Check user message - user_msg = chat_history[0] - assert user_msg.role == "USER" - assert user_msg.message == "What's the weather like in Tokyo?" - - # Check assistant message with tool calls - assistant_msg = chat_history[1] + assistant_msg = chat_history[0] assert assistant_msg.role == "CHATBOT" assert assistant_msg.message == "I will look up the weather in Tokyo." assert assistant_msg.toolCalls is not None assert len(assistant_msg.toolCalls) == 1 assert assistant_msg.toolCalls[0].name == "get_weather" - # The parameters should be parsed as JSON assert assistant_msg.toolCalls[0].parameters == {"location": "Tokyo"} - # Note: The tool message (last message) is excluded from chat history - # This is the expected behavior for Cohere models + tool_msg = chat_history[1] + assert tool_msg.role == "TOOL" + assert tool_msg.toolResults[0].call.name == "get_weather" + assert tool_msg.toolResults[0].call.parameters == {"location": "Tokyo"} + assert ( + tool_msg.toolResults[0].outputs[0]["output"] + == "The weather in Tokyo is 22°C with partly cloudy skies." + ) def test_cohere_streaming_chunk_handling(self): """Test Cohere streaming chunk handling""" @@ -457,7 +462,7 @@ class TestOCICohereToolCalls: assert "tool_choice" not in supported_params def test_cohere_default_parameters(self): - """Test that Cohere requests include required default parameters""" + """Test that Cohere requests do not inject hardcoded defaults — caller supplies all params.""" config = OCIChatConfig() messages = [{"role": "user", "content": "Hello"}] optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} @@ -472,12 +477,11 @@ class TestOCICohereToolCalls: chat_request = transformed_request["chatRequest"] - # Verify all required default parameters are present - assert chat_request["maxTokens"] == 600 - assert chat_request["temperature"] == 1 - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 + # No hardcoded defaults injected — only pass through what the user supplies + assert "maxTokens" not in chat_request + assert "topK" not in chat_request + assert "topP" not in chat_request + assert "frequencyPenalty" not in chat_request def test_cohere_parameter_override(self): """Test that user-provided parameters override defaults""" @@ -499,14 +503,104 @@ class TestOCICohereToolCalls: chat_request = transformed_request["chatRequest"] - # Verify user parameters override defaults + # Verify user parameters are passed through assert chat_request["temperature"] == 0.5 assert chat_request["maxTokens"] == 1000 - # Verify other defaults are still present - assert chat_request["topK"] == 0 - assert chat_request["topP"] == 0.75 - assert chat_request["frequencyPenalty"] == 0 + # Unset params are absent (no hardcoded defaults) + assert "topK" not in chat_request + assert "topP" not in chat_request + assert "frequencyPenalty" not in chat_request + + def test_cohere_response_finish_reason_tool_call(self): + """Test that finishReason='TOOL_CALL' is accepted by Pydantic and mapped to 'tool_calls'.""" + config = OCIChatConfig() + + mock_cohere_response = { + "modelId": "cohere.command-latest", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "", + "finishReason": "TOOL_CALL", + "toolCalls": [ + {"name": "get_weather", "parameters": {"location": "London"}} + ], + "usage": { + "promptTokens": 20, + "completionTokens": 10, + "totalTokens": 30, + }, + }, + } + + response = httpx.Response( + status_code=200, + json=mock_cohere_response, + headers={"Content-Type": "application/json"}, + ) + + result = config.transform_response( + model="cohere.command-latest", + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.tool_calls is not None + assert len(result.choices[0].message.tool_calls) == 1 + assert result.choices[0].message.tool_calls[0].function.name == "get_weather" + + def test_cohere_response_unknown_finish_reason_degrades_to_stop(self): + """A future/unknown finishReason in non-streaming responses must + degrade to ``stop`` via ``handle_cohere_response``'s fallback + rather than crash Pydantic validation. Mirrors the streaming + handler's behavior. See bug caf74429. + """ + config = OCIChatConfig() + + mock_cohere_response = { + "modelId": "cohere.command-latest", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "hello", + "finishReason": "FUTURE_REASON_NOT_YET_KNOWN", + "usage": { + "promptTokens": 1, + "completionTokens": 1, + "totalTokens": 2, + }, + }, + } + + response = httpx.Response( + status_code=200, + json=mock_cohere_response, + headers={"Content-Type": "application/json"}, + ) + + result = config.transform_response( + model="cohere.command-latest", + raw_response=response, + model_response=ModelResponse(), + logging_obj={}, # type: ignore + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding={}, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].finish_reason == "stop" def test_cohere_vendor_detection(self): """Test that Cohere models are correctly identified""" @@ -532,7 +626,7 @@ class TestOCICohereToolCalls: ] # The function should handle missing function key gracefully - cohere_tools = config.adapt_tool_definitions_to_cohere_standard(invalid_tools) + cohere_tools = adapt_tool_definitions_to_cohere_standard(invalid_tools) # Should create a tool with empty name and description assert len(cohere_tools) == 1 @@ -686,16 +780,114 @@ class TestOCICoherePreambleOverride: {"role": "assistant", "content": "First answer"}, {"role": "user", "content": "Second question"}, ] + optional_params = {"oci_compartment_id": TEST_COMPARTMENT_ID} - chat_history = config.adapt_messages_to_cohere_standard(messages) + result = config.transform_request( + model="cohere.command-latest", + messages=messages, # type: ignore + optional_params=optional_params, + litellm_params={}, + headers={}, + ) - # Should contain user and assistant only, no system - # Note: adapt_messages_to_cohere_standard excludes the last message - roles = [msg.role for msg in chat_history] + chat_request = result["chatRequest"] + roles = [msg["role"] for msg in chat_request["chatHistory"]] assert "SYSTEM" not in roles assert roles == ["USER", "CHATBOT"] +class TestCohereStreamChunkEdgeCases: + """Additional coverage for handle_cohere_stream_chunk error/edge paths.""" + + def _wrapper(self): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + return OCIStreamWrapper( + completion_stream=MagicMock(), + model="cohere.command-latest", + logging_obj=MagicMock(), + ) + + def test_stream_chunk_tool_call_finish_reason(self): + wrapper = self._wrapper() + chunk = { + "apiFormat": "COHERE", + "text": "", + "index": 0, + "finishReason": "TOOL_CALL", + } + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].finish_reason == "tool_calls" + + def test_stream_chunk_max_tokens_finish_reason(self): + wrapper = self._wrapper() + chunk = { + "apiFormat": "COHERE", + "text": "truncated", + "index": 0, + "finishReason": "MAX_TOKENS", + } + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].finish_reason == "length" + + def test_stream_chunk_unknown_finish_reason_does_not_raise(self): + from litellm.llms.oci.chat.cohere import handle_cohere_stream_chunk + + chunk = { + "apiFormat": "COHERE", + "text": "", + "index": 0, + "finishReason": "FUTURE_REASON", + } + # Should not raise — unknown reasons fall through the elif chain unchanged + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0] is not None + + def test_stream_chunk_null_index_defaults_to_zero(self): + wrapper = self._wrapper() + chunk = {"apiFormat": "COHERE", "text": "hi", "index": None} + result = wrapper.chunk_creator(f"data: {json.dumps(chunk)}") + assert result.choices[0].index == 0 + + +class TestCohereMessageAdaptationEdgeCases: + """Coverage for adapt_messages_to_cohere_standard error paths.""" + + def test_json_decode_error_in_tool_args_defaults_to_empty(self): + from litellm.llms.oci.chat.cohere import adapt_messages_to_cohere_standard + + messages = [ + { + "role": "assistant", + "content": "calling", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "fn", "arguments": "NOT JSON {{{"}, + } + ], + }, + {"role": "user", "content": "follow up"}, + ] + # Should not raise — bad JSON defaults to empty params {} + history = adapt_messages_to_cohere_standard(messages) + assert history[0].toolCalls[0].parameters == {} + + def test_extract_text_content_list_with_non_dict_items(self): + from litellm.llms.oci.chat.cohere import _extract_text_content + + # List with a non-dict item — should be silently skipped + result = _extract_text_content([{"type": "text", "text": "hello"}, "bad_item"]) + assert result == "hello" + + def test_extract_text_content_non_string_non_list(self): + from litellm.llms.oci.chat.cohere import _extract_text_content + + result = _extract_text_content(12345) + assert result == "12345" + + class TestOCICohereStreaming: """Test Cohere streaming functionality""" @@ -713,9 +905,9 @@ class TestOCICohereStreaming: """Test OCIStreamWrapper initialization""" stream_wrapper = self._create_stream_wrapper() + # chunk_creator is the public dispatch entry point assert hasattr(stream_wrapper, "chunk_creator") - assert hasattr(stream_wrapper, "_handle_cohere_stream_chunk") - assert hasattr(stream_wrapper, "_handle_generic_stream_chunk") + assert callable(stream_wrapper.chunk_creator) def test_cohere_streaming_chunk_parsing(self): """Test parsing of Cohere streaming chunks""" @@ -739,10 +931,12 @@ class TestOCICohereStreaming: def test_cohere_streaming_non_json_chunk(self): """Test error handling for non-JSON chunk""" + from litellm.llms.oci.common_utils import OCIError + stream_wrapper = self._create_stream_wrapper() # Test non-JSON chunk - with pytest.raises(json.JSONDecodeError): + with pytest.raises(OCIError, match="Chunk cannot be parsed as JSON"): stream_wrapper.chunk_creator("data: invalid json") def test_cohere_streaming_generic_chunk_fallback(self): diff --git a/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py new file mode 100644 index 0000000000..7583e3bc18 --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_generic_chat.py @@ -0,0 +1,440 @@ +""" +Unit tests for litellm/llms/oci/chat/generic.py — error paths and stream handling. +""" + +import json +import pytest +from unittest.mock import MagicMock + +import httpx + +from litellm import ModelResponse +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_messages_to_generic_oci_standard_content_message, + adapt_messages_to_generic_oci_standard_tool_call, + handle_generic_response, + handle_generic_stream_chunk, +) +from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIStreamWrapper +from litellm.llms.oci.common_utils import OCIError + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard_content_message — error paths +# --------------------------------------------------------------------------- + + +class TestGenericContentMessageErrors: + def test_non_dict_content_item_raises(self): + with pytest.raises(OCIError, match="must be a dictionary"): + adapt_messages_to_generic_oci_standard_content_message( + "user", ["not a dict"] + ) + + def test_non_string_type_field_raises(self): + with pytest.raises(OCIError, match="string `type` field"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": 123, "text": "hi"}] + ) + + def test_unsupported_content_type_raises(self): + with pytest.raises(OCIError, match="not supported by OCI"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "video_url", "url": "https://example.com/v.mp4"}] + ) + + def test_non_string_text_raises(self): + with pytest.raises(OCIError, match="must have a string `text` field"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "text", "text": 42}] + ) + + def test_image_url_as_invalid_type_raises(self): + with pytest.raises(OCIError, match="must be a string or an object"): + adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "image_url", "image_url": 99}] + ) + + def test_image_url_as_string(self): + msg = adapt_messages_to_generic_oci_standard_content_message( + "user", [{"type": "image_url", "image_url": "https://example.com/img.png"}] + ) + assert msg.content[0].imageUrl.url == "https://example.com/img.png" + + def test_image_url_as_dict(self): + msg = adapt_messages_to_generic_oci_standard_content_message( + "user", + [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.png"}, + } + ], + ) + assert msg.content[0].imageUrl.url == "https://example.com/img.png" + + def test_text_content_string(self): + msg = adapt_messages_to_generic_oci_standard_content_message("user", "hello") + assert msg.content[0].text == "hello" + + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard_tool_call — error paths +# --------------------------------------------------------------------------- + + +class TestGenericToolCallErrors: + def test_non_dict_tool_call_raises(self): + with pytest.raises(OCIError, match="must be a dictionary"): + adapt_messages_to_generic_oci_standard_tool_call("assistant", ["bad"]) + + def test_non_function_type_raises(self): + with pytest.raises(OCIError, match="only supports function tool calls"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "database", + "id": "x", + "function": {"name": "f", "arguments": "{}"}, + } + ], + ) + + def test_non_string_id_raises(self): + with pytest.raises(OCIError, match="id.*must be a string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": 123, + "function": {"name": "f", "arguments": "{}"}, + } + ], + ) + + def test_non_dict_function_raises(self): + with pytest.raises(OCIError, match="`function` must be a dictionary"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [{"type": "function", "id": "c1", "function": "not_a_dict"}], + ) + + def test_non_string_function_name_raises(self): + with pytest.raises(OCIError, match="function.name.*must be a string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": "c1", + "function": {"name": 5, "arguments": "{}"}, + } + ], + ) + + def test_non_string_arguments_raises(self): + with pytest.raises(OCIError, match="arguments.*must be a JSON string"): + adapt_messages_to_generic_oci_standard_tool_call( + "assistant", + [ + { + "type": "function", + "id": "c1", + "function": {"name": "fn", "arguments": {"key": "val"}}, + } + ], + ) + + +# --------------------------------------------------------------------------- +# adapt_messages_to_generic_oci_standard — combined paths +# --------------------------------------------------------------------------- + + +class TestGenericMessageAdaptation: + def test_tool_calls_not_list_raises(self): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": "not_a_list", + } + ] + with pytest.raises(OCIError, match="`tool_calls` must be a list"): + adapt_messages_to_generic_oci_standard(messages) + + def test_tool_result_non_string_tool_call_id_raises(self): + messages = [{"role": "tool", "content": "result", "tool_call_id": 999}] + with pytest.raises(OCIError, match="string `tool_call_id`"): + adapt_messages_to_generic_oci_standard(messages) + + def test_tool_result_non_string_content_raises(self): + messages = [ + {"role": "tool", "content": {"structured": "data"}, "tool_call_id": "c1"} + ] + with pytest.raises(OCIError, match="`content` must be a string"): + adapt_messages_to_generic_oci_standard(messages) + + def test_non_string_non_list_content_raises(self): + messages = [{"role": "user", "content": 42}] + with pytest.raises(OCIError, match="`content` must be a string or list"): + adapt_messages_to_generic_oci_standard(messages) + + +# --------------------------------------------------------------------------- +# handle_generic_response — error and None message paths +# --------------------------------------------------------------------------- + + +class TestHandleGenericResponse: + def _make_response(self, body: dict, status: int = 200) -> httpx.Response: + return httpx.Response(status_code=status, json=body) + + def _valid_body(self, message=None): + return { + "modelId": "xai.grok-4", + "modelVersion": "1", + "chatResponse": { + "apiFormat": "GENERIC", + "timeCreated": "2024-01-01T00:00:00Z", + "choices": [ + {"message": message, "finishReason": "COMPLETE", "index": 0} + ], + "usage": {"promptTokens": 5, "completionTokens": 5, "totalTokens": 10}, + }, + } + + def test_none_response_message(self): + body = self._valid_body(message=None) + raw = self._make_response(body) + # Should not raise — None message means no content set + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.model == "xai.grok-4" + + def test_response_with_text_content(self): + body = self._valid_body( + message={ + "role": "ASSISTANT", + "content": [{"type": "TEXT", "text": "Hello!"}], + } + ) + raw = self._make_response(body) + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.choices[0].message.content == "Hello!" + + def test_response_with_tool_calls(self): + body = self._valid_body( + message={ + "role": "ASSISTANT", + "content": [], + "toolCalls": [ + { + "id": "call_abc", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Tokyo"}', + } + ], + } + ) + raw = self._make_response(body) + result = handle_generic_response(body, "xai.grok-4", ModelResponse(), raw) + assert result.choices[0].message.tool_calls is not None + + +# --------------------------------------------------------------------------- +# handle_generic_stream_chunk — finish reasons and error paths +# --------------------------------------------------------------------------- + + +class TestHandleGenericStreamChunk: + def test_max_tokens_finish_reason(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "MAX_TOKENS"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "length" + + def test_tool_calls_finish_reason(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "TOOL_CALLS"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + + def test_unknown_finish_reason_does_not_raise(self): + chunk = {"apiFormat": "GENERIC", "index": 0, "finishReason": "SOME_NEW_REASON"} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0] is not None + + def test_null_index_defaults_to_zero(self): + chunk = {"apiFormat": "GENERIC", "index": None, "finishReason": None} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].index == 0 + + def test_image_content_in_stream_raises(self): + from litellm.types.llms.oci import OCIImageContentPart, OCIImageUrl, OCIMessage + + chunk = { + "apiFormat": "GENERIC", + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [ + { + "type": "IMAGE", + "imageUrl": {"url": "https://example.com/img.png"}, + } + ], + }, + } + with pytest.raises(OCIError, match="image content"): + handle_generic_stream_chunk(chunk) + + def test_stream_chunk_with_tool_calls(self): + chunk = { + "apiFormat": "GENERIC", + "index": 0, + "message": { + "role": "ASSISTANT", + "content": [], + "toolCalls": [ + { + "id": "call_abc", + "type": "FUNCTION", + "name": "get_weather", + "arguments": '{"location": "Tokyo"}', + } + ], + }, + } + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.tool_calls is not None + + +# --------------------------------------------------------------------------- +# OCIStreamWrapper.chunk_creator — non-string chunk +# --------------------------------------------------------------------------- + + +class TestOCIStreamWrapperChunkCreator: + def _wrapper(self): + return OCIStreamWrapper( + completion_stream=MagicMock(), + model="xai.grok-4", + logging_obj=MagicMock(), + ) + + def test_non_string_chunk_raises(self): + w = self._wrapper() + with pytest.raises(ValueError, match="not a string"): + w.chunk_creator({"already": "parsed"}) + + +# --------------------------------------------------------------------------- +# GPT-5 family: maxCompletionTokens routing +# +# Regression guard: OCI rejects "maxTokens" for openai.gpt-5* models with HTTP +# 400 ("Use 'maxCompletionTokens' instead.") — verified against live OCI. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def _register_oci_gpt5_in_catalog(): + """Guarantee OCI GPT-5 catalog entries with supports_reasoning=True are + present for the duration of the test, regardless of whether + ``litellm.model_cost`` was populated from the bundled + ``model_prices_and_context_window.json`` (which ships them) or from a + remote map that may lag behind. + """ + import litellm + + needed = { + "oci/openai.gpt-5", + "oci/openai.gpt-5-mini", + "oci/openai.gpt-5-nano", + } + added = [] + for key in needed: + if key not in litellm.model_cost: + litellm.model_cost[key] = { + "litellm_provider": "oci", + "mode": "chat", + "supports_reasoning": True, + } + added.append(key) + yield + for key in added: + litellm.model_cost.pop(key, None) + + +class TestGpt5MaxCompletionTokens: + def test_helper_detects_gpt5_family(self, _register_oci_gpt5_in_catalog): + from litellm.llms.oci.chat.transformation import ( + _model_uses_max_completion_tokens, + ) + + assert _model_uses_max_completion_tokens("openai.gpt-5") is True + assert _model_uses_max_completion_tokens("openai.gpt-5-mini") is True + assert _model_uses_max_completion_tokens("openai.gpt-5-nano") is True + assert _model_uses_max_completion_tokens("oci/openai.gpt-5") is True + + assert _model_uses_max_completion_tokens("openai.gpt-oss-120b") is False + assert _model_uses_max_completion_tokens("meta.llama-3.3-70b-instruct") is False + assert _model_uses_max_completion_tokens("cohere.command-latest") is False + assert _model_uses_max_completion_tokens("") is False + + def test_gpt5_routes_max_tokens_to_max_completion_tokens( + self, _register_oci_gpt5_in_catalog + ): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + # Both shapes optional_params can take after upstream map_openai_params: + # 1. openai-side key still present + out_a = cfg._get_optional_params( + OCIVendors.GENERIC, {"max_tokens": 64}, model="openai.gpt-5" + ) + assert out_a.get("maxCompletionTokens") == 64 + assert "maxTokens" not in out_a + + # 2. already pre-translated to OCI alias + out_b = cfg._get_optional_params( + OCIVendors.GENERIC, {"maxTokens": 64}, model="openai.gpt-5-mini" + ) + assert out_b.get("maxCompletionTokens") == 64 + assert "maxTokens" not in out_b + + def test_non_gpt5_keeps_max_tokens(self): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + out = cfg._get_optional_params( + OCIVendors.GENERIC, + {"max_tokens": 64}, + model="meta.llama-3.3-70b-instruct", + ) + assert out.get("maxTokens") == 64 + assert "maxCompletionTokens" not in out + + def test_cohere_reasoning_model_keeps_max_tokens(self): + from litellm.llms.oci.chat.transformation import OCIChatConfig, OCIVendors + + cfg = OCIChatConfig() + out = cfg._get_optional_params( + OCIVendors.COHERE, + {"max_tokens": 64}, + model="cohere.command-a-reasoning", + ) + assert out.get("maxTokens") == 64 + assert "maxCompletionTokens" not in out + + def test_payload_serializes_max_completion_tokens(self): + from litellm.types.llms.oci import OCIChatRequestPayload + + payload = OCIChatRequestPayload( + apiFormat="GENERIC", + messages=[], + maxCompletionTokens=64, + ) + dumped = payload.model_dump(exclude_none=True) + assert dumped["maxCompletionTokens"] == 64 + assert "maxTokens" not in dumped diff --git a/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py b/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py new file mode 100644 index 0000000000..a2faf664ee --- /dev/null +++ b/tests/test_litellm/llms/oci/chat/test_oci_sse_splitter.py @@ -0,0 +1,232 @@ +""" +Tests for the OCI SSE event splitter. + +Regression coverage for the streaming bug John Lathouwers reported: the old +``split_chunks`` helper split each individual HTTP read on ``\\n\\n``, so any +event that straddled a read boundary or any pair of events separated by a +single ``\\n`` would yield malformed chunks to ``OCIStreamWrapper.chunk_creator`` +and crash ``json.loads``. +""" + +import asyncio +from typing import AsyncIterator, Iterator, List + +from litellm.llms.oci.chat.transformation import ( + _aiter_sse_events, + _iter_sse_events, +) + + +def _collect_sync(stream: Iterator[str]) -> List[str]: + return list(_iter_sse_events(iter(stream))) + + +def _collect_async(chunks: List[str]) -> List[str]: + async def _src() -> AsyncIterator[str]: + for c in chunks: + yield c + + async def _run() -> List[str]: + out: List[str] = [] + async for line in _aiter_sse_events(_src()): + out.append(line) + return out + + return asyncio.run(_run()) + + +# --------------------------------------------------------------------------- +# Sync splitter +# --------------------------------------------------------------------------- + + +class TestIterSseEventsSync: + def test_well_formed_double_newline_separators(self): + reads = ['data: {"a":1}\n\ndata: {"a":2}\n\n'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_event_split_across_two_reads(self): + # The bug: read 1 ends mid-JSON, read 2 finishes it. Old code would + # have yielded a truncated 'data: {"index":0,"text":"hel' and crashed + # json.loads in chunk_creator. + reads = [ + 'data: {"index":0,"text":"hel', + 'lo"}\n\n', + ] + assert _collect_sync(reads) == ['data: {"index":0,"text":"hello"}'] + + def test_event_split_into_many_tiny_reads(self): + full = 'data: {"k":"value with spaces"}\n\n' + reads = [full[i : i + 3] for i in range(0, len(full), 3)] + assert _collect_sync(reads) == ['data: {"k":"value with spaces"}'] + + def test_single_newline_separator(self): + # The other shape John saw: events separated by just '\n'. + reads = ['data: {"a":1}\ndata: {"a":2}\ndata: {"a":3}\n'] + assert _collect_sync(reads) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_mixed_separators_in_one_read(self): + reads = ['data: {"a":1}\ndata: {"a":2}\n\ndata: {"a":3}\n\n'] + assert _collect_sync(reads) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_keepalive_and_comment_lines_dropped(self): + # SSE keepalives ("\n") and comment lines (": ping") must not be + # forwarded to chunk_creator, which would reject anything not + # starting with 'data:'. + reads = [ + "\n", + ": ping\n", + 'data: {"a":1}\n\n', + "\n", + ": keepalive\n\n", + 'data: {"a":2}\n\n', + ] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_partial_event_flushed_at_eof(self): + # Final event arrives without a terminating newline. The splitter + # must still emit it once the upstream iterator is exhausted. + reads = ['data: {"a":1}\n\n', 'data: {"a":2}'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_non_data_line_dropped_at_eof(self): + reads = ['data: {"a":1}\n\n: trailing-comment'] + assert _collect_sync(reads) == ['data: {"a":1}'] + + def test_empty_stream(self): + assert _collect_sync([]) == [] + + def test_only_whitespace_and_keepalives(self): + assert _collect_sync(["\n", "\n\n", ": ping\n"]) == [] + + def test_boundary_between_data_keyword_and_payload(self): + # The 'data:' marker itself straddles a read boundary. + reads = ["dat", 'a: {"a":1}\n\n'] + assert _collect_sync(reads) == ['data: {"a":1}'] + + def test_carriage_return_in_payload_preserved(self): + # SSE-over-the-wire may use \r\n line endings. We split on \n; the + # \r ends up on the previous line and strip() removes it. + reads = ['data: {"a":1}\r\ndata: {"a":2}\r\n'] + assert _collect_sync(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + +# --------------------------------------------------------------------------- +# Async splitter — same scenarios, parallel coverage +# --------------------------------------------------------------------------- + + +class TestIterSseEventsAsync: + def test_well_formed_double_newline_separators(self): + assert _collect_async(['data: {"a":1}\n\ndata: {"a":2}\n\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + def test_event_split_across_two_reads(self): + assert _collect_async(['data: {"index":0,"text":"hel', 'lo"}\n\n']) == [ + 'data: {"index":0,"text":"hello"}' + ] + + def test_event_split_into_many_tiny_reads(self): + full = 'data: {"k":"value with spaces"}\n\n' + reads = [full[i : i + 3] for i in range(0, len(full), 3)] + assert _collect_async(reads) == ['data: {"k":"value with spaces"}'] + + def test_single_newline_separator(self): + assert _collect_async(['data: {"a":1}\ndata: {"a":2}\ndata: {"a":3}\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + 'data: {"a":3}', + ] + + def test_mixed_separators_in_one_read(self): + assert _collect_async( + ['data: {"a":1}\ndata: {"a":2}\n\ndata: {"a":3}\n\n'] + ) == ['data: {"a":1}', 'data: {"a":2}', 'data: {"a":3}'] + + def test_keepalive_and_comment_lines_dropped(self): + reads = [ + "\n", + ": ping\n", + 'data: {"a":1}\n\n', + "\n", + ": keepalive\n\n", + 'data: {"a":2}\n\n', + ] + assert _collect_async(reads) == ['data: {"a":1}', 'data: {"a":2}'] + + def test_trailing_partial_event_flushed_at_eof(self): + assert _collect_async(['data: {"a":1}\n\n', 'data: {"a":2}']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + def test_trailing_non_data_line_dropped_at_eof(self): + assert _collect_async(['data: {"a":1}\n\n: trailing-comment']) == [ + 'data: {"a":1}' + ] + + def test_empty_stream(self): + assert _collect_async([]) == [] + + def test_only_whitespace_and_keepalives(self): + assert _collect_async(["\n", "\n\n", ": ping\n"]) == [] + + def test_boundary_between_data_keyword_and_payload(self): + assert _collect_async(["dat", 'a: {"a":1}\n\n']) == ['data: {"a":1}'] + + def test_carriage_return_in_payload_preserved(self): + assert _collect_async(['data: {"a":1}\r\ndata: {"a":2}\r\n']) == [ + 'data: {"a":1}', + 'data: {"a":2}', + ] + + +# --------------------------------------------------------------------------- +# End-to-end: feed an awkwardly-chunked stream into OCIStreamWrapper and +# verify chunk_creator still parses each yielded line. This is the smoke +# test that proves the integration with the downstream consumer holds. +# --------------------------------------------------------------------------- + + +class TestSseSplitterFeedsChunkCreator: + def test_split_event_parses_cleanly(self): + # Build a realistic GENERIC OCI streaming payload, then chop it into + # awkward reads. The splitter must reassemble exactly one event so + # json.loads inside chunk_creator does not raise. + import json + from unittest.mock import MagicMock + + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + payload = { + "apiFormat": "GENERIC", + "message": {"content": [{"text": "hello"}]}, + "finishReason": None, + } + wire = f"data: {json.dumps(payload)}\n\n" + # Split the wire string at an awkward point inside the JSON body. + cut = wire.index('"hello"') + 3 + reads = [wire[:cut], wire[cut:]] + + # Drive the splitter directly and confirm we get exactly one event. + events = list(_iter_sse_events(iter(reads))) + assert len(events) == 1 + assert events[0].startswith("data: ") + # chunk_creator should now parse this without raising. + wrapper = OCIStreamWrapper( + completion_stream=MagicMock(), + model="xai.grok-4", + logging_obj=MagicMock(), + ) + # Must not raise. + wrapper.chunk_creator(events[0]) diff --git a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py index f9d4be8032..acad5da93e 100644 --- a/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py +++ b/tests/test_litellm/llms/oci/chat/test_oci_streaming_tool_calls.py @@ -11,13 +11,10 @@ Error: ValidationError: 1 validation error for OCIStreamChunk message.toolCalls. import os import sys -import pytest -from unittest.mock import MagicMock -# Adds the parent directory to the system path sys.path.insert(0, os.path.abspath("../../../../..")) -from litellm.llms.oci.chat.transformation import OCIStreamWrapper +from litellm.llms.oci.chat.generic import handle_generic_stream_chunk from litellm.types.utils import ModelResponseStream @@ -26,12 +23,9 @@ class TestOCIStreamingToolCalls: def test_stream_chunk_with_missing_arguments_field(self): """ - Test that streaming chunks with tool calls missing 'arguments' field are handled. - OCI API can return tool calls in early chunks without the 'arguments' field, which should be filled with an empty string to satisfy Pydantic validation. """ - # Mock streaming chunk with tool call missing 'arguments' field chunk_data = { "index": 0, "finishReason": None, @@ -43,21 +37,13 @@ class TestOCIStreamingToolCalls: "type": "FUNCTION", "id": "call_abc123", "name": "get_weather", - # Note: 'arguments' field is missing + # 'arguments' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - # This should not raise a ValidationError - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert len(result.choices) == 1 @@ -66,9 +52,7 @@ class TestOCIStreamingToolCalls: assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" def test_stream_chunk_with_missing_id_field(self): - """ - Test that streaming chunks with tool calls missing 'id' field are handled. - """ + """Missing 'id' gets a generated call_* id.""" chunk_data = { "index": 0, "finishReason": None, @@ -80,29 +64,20 @@ class TestOCIStreamingToolCalls: "type": "FUNCTION", "name": "get_weather", "arguments": '{"location": "San Francisco"}', - # Note: 'id' field is missing + # 'id' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None - assert result.choices[0].delta.tool_calls[0]["id"] == "" + assert result.choices[0].delta.tool_calls[0]["id"].startswith("call_") def test_stream_chunk_with_missing_name_field(self): - """ - Test that streaming chunks with tool calls missing 'name' field are handled. - """ + """Missing 'name' defaults to empty string.""" chunk_data = { "index": 0, "finishReason": None, @@ -114,29 +89,20 @@ class TestOCIStreamingToolCalls: "type": "FUNCTION", "id": "call_abc123", "arguments": '{"location": "San Francisco"}', - # Note: 'name' field is missing + # 'name' field is missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None assert result.choices[0].delta.tool_calls[0]["function"]["name"] == "" def test_stream_chunk_with_all_missing_fields(self): - """ - Test that streaming chunks with tool calls missing all optional fields are handled. - """ + """All optional fields missing — all default gracefully.""" chunk_data = { "index": 0, "finishReason": None, @@ -146,31 +112,22 @@ class TestOCIStreamingToolCalls: "toolCalls": [ { "type": "FUNCTION" - # All fields missing: id, name, arguments + # id, name, arguments all missing } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None - assert result.choices[0].delta.tool_calls[0]["id"] == "" + assert result.choices[0].delta.tool_calls[0]["id"].startswith("call_") assert result.choices[0].delta.tool_calls[0]["function"]["name"] == "" assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" def test_stream_chunk_with_complete_tool_call(self): - """ - Test that streaming chunks with complete tool calls still work correctly. - """ + """Fully-populated tool call passes through unchanged.""" chunk_data = { "index": 0, "finishReason": None, @@ -188,14 +145,7 @@ class TestOCIStreamingToolCalls: }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.tool_calls is not None @@ -210,10 +160,64 @@ class TestOCIStreamingToolCalls: ) def test_stream_chunk_with_multiple_tool_calls_missing_fields(self): - """ - Test that streaming chunks with multiple tool calls, some with missing fields, are handled. - """ + """Multiple tool calls with a mix of complete and incomplete entries.""" chunk_data = { + "index": 0, + "finishReason": None, + "message": { + "role": "ASSISTANT", + "content": None, + "toolCalls": [ + {"type": "FUNCTION", "id": "call_1", "name": "get_weather"}, + { + "type": "FUNCTION", + "name": "get_time", + "arguments": '{"timezone": "UTC"}', + }, + { + "type": "FUNCTION", + "id": "call_3", + "name": "calculate", + "arguments": '{"expression": "2+2"}', + }, + ], + }, + } + + result = handle_generic_stream_chunk(chunk_data) + + assert isinstance(result, ModelResponseStream) + assert result.choices[0].delta.tool_calls is not None + assert len(result.choices[0].delta.tool_calls) == 3 + + assert result.choices[0].delta.tool_calls[0]["id"] == "call_1" + assert ( + result.choices[0].delta.tool_calls[0]["function"]["name"] == "get_weather" + ) + assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" + + assert result.choices[0].delta.tool_calls[1]["id"].startswith("call_") + assert result.choices[0].delta.tool_calls[1]["function"]["name"] == "get_time" + assert ( + result.choices[0].delta.tool_calls[1]["function"]["arguments"] + == '{"timezone": "UTC"}' + ) + + assert result.choices[0].delta.tool_calls[2]["id"] == "call_3" + assert result.choices[0].delta.tool_calls[2]["function"]["name"] == "calculate" + assert ( + result.choices[0].delta.tool_calls[2]["function"]["arguments"] + == '{"expression": "2+2"}' + ) + + def test_stream_chunk_missing_id_is_deterministic_across_chunks(self): + """ + Two chunks emitting the same logical tool call (same name + arguments + at the same position) must receive the *same* synthesized id so the + downstream stream-merger does not treat them as distinct calls. + Random uuid4 per chunk would regress this — see bug ffdef760. + """ + same_chunk_payload = lambda: { "index": 0, "finishReason": None, "message": { @@ -222,67 +226,25 @@ class TestOCIStreamingToolCalls: "toolCalls": [ { "type": "FUNCTION", - "id": "call_1", "name": "get_weather", - # Missing arguments - }, - { - "type": "FUNCTION", - "name": "get_time", - "arguments": '{"timezone": "UTC"}', - # Missing id - }, - { - "type": "FUNCTION", - "id": "call_3", - "name": "calculate", - "arguments": '{"expression": "2+2"}', - # Complete - }, + "arguments": '{"location": "San Francisco"}', + } ], }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) + first = handle_generic_stream_chunk(same_chunk_payload()) + second = handle_generic_stream_chunk(same_chunk_payload()) - result = wrapper._handle_generic_stream_chunk(chunk_data) - - assert isinstance(result, ModelResponseStream) - assert result.choices[0].delta.tool_calls is not None - assert len(result.choices[0].delta.tool_calls) == 3 - - # First tool call - missing arguments - assert result.choices[0].delta.tool_calls[0]["id"] == "call_1" - assert ( - result.choices[0].delta.tool_calls[0]["function"]["name"] == "get_weather" - ) - assert result.choices[0].delta.tool_calls[0]["function"]["arguments"] == "" - - # Second tool call - missing id - assert result.choices[0].delta.tool_calls[1]["id"] == "" - assert result.choices[0].delta.tool_calls[1]["function"]["name"] == "get_time" - assert ( - result.choices[0].delta.tool_calls[1]["function"]["arguments"] - == '{"timezone": "UTC"}' - ) - - # Third tool call - complete - assert result.choices[0].delta.tool_calls[2]["id"] == "call_3" - assert result.choices[0].delta.tool_calls[2]["function"]["name"] == "calculate" - assert ( - result.choices[0].delta.tool_calls[2]["function"]["arguments"] - == '{"expression": "2+2"}' - ) + assert first.choices[0].delta.tool_calls is not None + assert second.choices[0].delta.tool_calls is not None + first_id = first.choices[0].delta.tool_calls[0]["id"] + second_id = second.choices[0].delta.tool_calls[0]["id"] + assert first_id == second_id + assert first_id.startswith("call_") def test_stream_chunk_without_tool_calls(self): - """ - Test that streaming chunks without tool calls continue to work as before. - """ + """Plain text chunks (no tool calls) pass through correctly.""" chunk_data = { "index": 0, "finishReason": None, @@ -292,14 +254,7 @@ class TestOCIStreamingToolCalls: }, } - wrapper = OCIStreamWrapper( - completion_stream=iter([]), - model="meta.llama-3.1-405b-instruct", - custom_llm_provider="oci", - logging_obj=MagicMock(), - ) - - result = wrapper._handle_generic_stream_chunk(chunk_data) + result = handle_generic_stream_chunk(chunk_data) assert isinstance(result, ModelResponseStream) assert result.choices[0].delta.content == "Hello, how can I help you?" diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py new file mode 100644 index 0000000000..30f49bea34 --- /dev/null +++ b/tests/test_litellm/llms/oci/embed/test_oci_embed_transformation.py @@ -0,0 +1,406 @@ +""" +Unit tests for OCI Generative AI embedding transformation. + +These tests exercise the transformation layer only — no real OCI calls are made. +""" + +import json +import os +import sys +from typing import Any +from unittest.mock import MagicMock + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.oci.common_utils import OCIError +from litellm.llms.oci.embed.transformation import OCI_EMBED_BATCH_LIMIT, OCIEmbedConfig +from litellm.types.utils import EmbeddingResponse, Usage + +# --------------------------------------------------------------------------- +# Test fixtures +# --------------------------------------------------------------------------- + +COMPARTMENT_ID = "ocid1.compartment.oc1..test" +BASE_PARAMS = { + "oci_region": "us-ashburn-1", + "oci_user": "ocid1.user.oc1..test", + "oci_fingerprint": "aa:bb:cc:dd", + "oci_tenancy": "ocid1.tenancy.oc1..test", + "oci_compartment_id": COMPARTMENT_ID, + "oci_key": "-----BEGIN RSA PRIVATE KEY-----\nfakekey\n-----END RSA PRIVATE KEY-----", +} + + +class TestOCIEmbedConfig: + def _config(self) -> OCIEmbedConfig: + return OCIEmbedConfig() + + # ------------------------------------------------------------------ + # validate_environment + # ------------------------------------------------------------------ + + def test_validate_environment_sets_headers(self): + cfg = self._config() + headers = cfg.validate_environment( + headers={}, + model="oci/cohere.embed-v3.0", + messages=[], + optional_params=BASE_PARAMS, + litellm_params={}, + ) + assert headers["content-type"] == "application/json" + assert "litellm/" in headers["user-agent"] + + # ------------------------------------------------------------------ + # get_complete_url + # ------------------------------------------------------------------ + + def test_get_complete_url_default_region(self): + cfg = self._config() + url = cfg.get_complete_url( + api_base=None, + api_key=None, + model="cohere.embed-v3.0", + optional_params={"oci_region": "us-chicago-1"}, + litellm_params={}, + ) + assert ( + url + == "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText" + ) + + def test_get_complete_url_respects_api_base(self): + """api_base is treated as a base URL — the action path is appended.""" + cfg = self._config() + url = cfg.get_complete_url( + api_base="https://custom.endpoint.example.com", + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.example.com/20231130/actions/embedText" + + def test_get_complete_url_strips_trailing_slash(self): + """Trailing slash is stripped from api_base before appending the action path.""" + cfg = self._config() + url = cfg.get_complete_url( + api_base="https://custom.endpoint.example.com/", + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.example.com/20231130/actions/embedText" + + def test_get_complete_url_full_url_is_not_doubled(self): + """A fully-formed embedText URL must not have the action path appended twice.""" + cfg = self._config() + full_url = ( + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + "/20231130/actions/embedText" + ) + url = cfg.get_complete_url( + api_base=full_url, + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, + litellm_params={}, + ) + assert url == full_url + + # ------------------------------------------------------------------ + # transform_embedding_request + # ------------------------------------------------------------------ + + def test_transform_request_single_string(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input="hello world", + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["compartmentId"] == COMPARTMENT_ID + assert result["servingMode"]["servingType"] == "ON_DEMAND" + assert result["servingMode"]["modelId"] == "cohere.embed-v3.0" + assert result["inputs"] == ["hello world"] + + def test_transform_request_list_of_texts(self): + cfg = self._config() + texts = ["hello", "world"] + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=texts, + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["inputs"] == texts + + def test_transform_request_with_input_type(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["query"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "input_type": "SEARCH_QUERY", + }, + headers={}, + ) + assert result["inputType"] == "SEARCH_QUERY" + + def test_transform_request_with_output_dimensions(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v4.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "outputDimensions": 512, + }, + headers={}, + ) + assert result["outputDimensions"] == 512 + + def test_transform_request_dedicated_serving_mode(self): + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "oci_serving_mode": "DEDICATED", + "oci_endpoint_id": "ocid1.genaiendpoint.oc1..test", + }, + headers={}, + ) + assert result["servingMode"]["servingType"] == "DEDICATED" + assert result["servingMode"]["endpointId"] == "ocid1.genaiendpoint.oc1..test" + assert "modelId" not in result["servingMode"] + + def test_transform_request_missing_compartment_id_raises(self): + cfg = self._config() + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "oci_compartment_id" in str(exc_info.value) + + def test_transform_request_batch_limit_exceeded_raises(self): + cfg = self._config() + texts = ["text"] * (OCI_EMBED_BATCH_LIMIT + 1) + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=texts, + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert str(OCI_EMBED_BATCH_LIMIT) in str(exc_info.value) + + def test_transform_request_invalid_serving_mode_raises(self): + cfg = self._config() + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["text"], + optional_params={ + "oci_compartment_id": COMPARTMENT_ID, + "oci_serving_mode": "INVALID", + }, + headers={}, + ) + assert exc_info.value.status_code == 400 + + def test_transform_request_none_input_becomes_string(self): + """Non-list, non-string inputs are coerced to str.""" + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=42, # type: ignore + optional_params={"oci_compartment_id": COMPARTMENT_ID}, + headers={}, + ) + assert result["inputs"] == ["42"] + + # ------------------------------------------------------------------ + # transform_embedding_response + # ------------------------------------------------------------------ + + def _mock_response(self, status_code: int, body: dict) -> httpx.Response: + return httpx.Response( + status_code=status_code, + content=json.dumps(body).encode(), + headers={"content-type": "application/json"}, + ) + + def test_transform_response_success(self): + cfg = self._config() + model_response = EmbeddingResponse() + raw = self._mock_response( + 200, + { + "embeddings": [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], + "modelId": "cohere.embed-v3.0", + "modelVersion": "3.0.0", + # Actual OCI API returns per-input token counts + "inputTextTokenCounts": [5, 5], + }, + ) + result = cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["index"] == 1 + assert result.model == "cohere.embed-v3.0" + assert result.usage.prompt_tokens == 10 + + def test_transform_response_no_usage(self): + cfg = self._config() + model_response = EmbeddingResponse() + raw = self._mock_response( + 200, + { + "embeddings": [[0.1]], + "modelId": "cohere.embed-v3.0", + "modelVersion": "3.0.0", + }, + ) + result = cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=model_response, + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert len(result.data) == 1 + + def test_transform_response_http_error_raises(self): + cfg = self._config() + raw = self._mock_response(401, {"error": "Unauthorized"}) + with pytest.raises(OCIError) as exc_info: + cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=EmbeddingResponse(), + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + assert exc_info.value.status_code == 401 + + def test_transform_response_invalid_json_raises(self): + cfg = self._config() + raw = httpx.Response( + status_code=200, + content=b"not-json", + headers={"content-type": "text/plain"}, + ) + with pytest.raises(OCIError): + cfg.transform_embedding_response( + model="cohere.embed-v3.0", + raw_response=raw, + model_response=EmbeddingResponse(), + logging_obj=MagicMock(), + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, + ) + + # ------------------------------------------------------------------ + # map_openai_params + # ------------------------------------------------------------------ + + def test_map_openai_params_dimensions(self): + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"dimensions": 512}, + optional_params={}, + model="cohere.embed-v4.0", + ) + assert result["outputDimensions"] == 512 + + def test_map_openai_params_encoding_format_not_supported(self): + """encoding_format is not a supported OCI param — it is silently ignored by map_openai_params. + + The litellm framework handles unsupported-param rejection above this layer, + based on get_supported_openai_params() not including 'encoding_format'. + """ + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"encoding_format": "float"}, + optional_params={}, + model="cohere.embed-v3.0", + ) + assert "encoding_format" not in result + + def test_map_openai_params_encoding_format_dropped_silently(self): + cfg = self._config() + result = cfg.map_openai_params( + non_default_params={"encoding_format": "float"}, + optional_params={}, + model="cohere.embed-v3.0", + drop_params=True, + ) + assert "encoding_format" not in result + + # ------------------------------------------------------------------ + # env var credential resolution + # ------------------------------------------------------------------ + + def test_env_var_compartment_id(self, monkeypatch): + monkeypatch.setenv("OCI_COMPARTMENT_ID", "ocid1.compartment.from.env") + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["hello"], + optional_params={}, # no compartment_id in params + headers={}, + ) + assert result["compartmentId"] == "ocid1.compartment.from.env" + + def test_explicit_param_overrides_env_var(self, monkeypatch): + monkeypatch.setenv("OCI_COMPARTMENT_ID", "ocid1.compartment.from.env") + cfg = self._config() + result = cfg.transform_embedding_request( + model="cohere.embed-v3.0", + input=["hello"], + optional_params={"oci_compartment_id": "ocid1.compartment.explicit"}, + headers={}, + ) + assert result["compartmentId"] == "ocid1.compartment.explicit" + + def test_env_var_region_used_in_url(self, monkeypatch): + monkeypatch.setenv("OCI_REGION", "eu-frankfurt-1") + cfg = self._config() + url = cfg.get_complete_url( + api_base=None, + api_key=None, + model="cohere.embed-v3.0", + optional_params={}, # no explicit region + litellm_params={}, + ) + assert "eu-frankfurt-1" in url diff --git a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py index 4ecca377e6..61c13ad62a 100644 --- a/tests/test_litellm/llms/oci/embed/test_oci_embedding.py +++ b/tests/test_litellm/llms/oci/embed/test_oci_embedding.py @@ -76,7 +76,7 @@ class TestOCIEmbeddingConfig: assert "embedText" in url def test_get_complete_url_custom_api_base(self): - """test_get_complete_url returns api_base as-is when provided.""" + """test_get_complete_url treats api_base as a base URL and appends the embedText path.""" config = OCIEmbeddingConfig() custom_base = "https://custom.oci.example.com/embed" url = config.get_complete_url( @@ -86,7 +86,7 @@ class TestOCIEmbeddingConfig: optional_params={}, litellm_params={}, ) - assert url == custom_base + assert url == f"{custom_base}/20231130/actions/embedText" def test_get_supported_openai_params(self): """test_get_supported_openai_params returns expected params list.""" @@ -96,7 +96,7 @@ class TestOCIEmbeddingConfig: assert "encoding_format" not in params def test_map_openai_params_dimensions(self): - """test dimensions is mapped correctly.""" + """test dimensions is mapped to outputDimensions (OCI API field name).""" config = OCIEmbeddingConfig() optional_params = {} result = config.map_openai_params( @@ -105,7 +105,8 @@ class TestOCIEmbeddingConfig: model=TEST_MODEL_NAME, drop_params=False, ) - assert result["dimensions"] == 512 + assert result["outputDimensions"] == 512 + assert "dimensions" not in result def test_validate_environment_with_credentials(self, supplied_params): """test validate_environment returns content-type and user-agent headers when credentials are supplied.""" @@ -122,13 +123,15 @@ class TestOCIEmbeddingConfig: assert "litellm" in result["user-agent"] def test_validate_environment_missing_credentials(self): - """test validate_environment raises Exception with 'Missing required parameters' when credentials are incomplete.""" + """test validate_environment raises OCIError when required credentials are missing.""" + from litellm.llms.oci.common_utils import OCIError + config = OCIEmbeddingConfig() incomplete_params = { "oci_user": "ocid1.user.oc1..xxx", # Missing oci_fingerprint, oci_tenancy, oci_key/oci_key_file, oci_compartment_id } - with pytest.raises(Exception) as excinfo: + with pytest.raises(OCIError, match="Missing required parameters"): config.validate_environment( headers={}, model=TEST_MODEL, @@ -136,7 +139,6 @@ class TestOCIEmbeddingConfig: optional_params=incomplete_params, litellm_params={}, ) - assert "Missing required parameters" in str(excinfo.value) def test_validate_environment_with_signer(self): """test validate_environment passes when oci_signer is provided.""" @@ -234,13 +236,15 @@ class TestOCIEmbeddingConfig: assert result["inputs"] == ["Hello world"] def test_transform_embedding_request_token_list_raises(self): - """test token-array inputs raise ValueError instead of silent conversion.""" + """test token-array inputs raise OCIError instead of silent conversion.""" + from litellm.llms.oci.common_utils import OCIError + config = OCIEmbeddingConfig() optional_params = { "oci_compartment_id": TEST_COMPARTMENT_ID, } with patch.object(config, "sign_request", return_value=({}, "{}")): - with pytest.raises(ValueError, match="does not support token-array"): + with pytest.raises(OCIError, match="does not support token-array"): config.transform_embedding_request( model=TEST_MODEL_NAME, input=[[1234, 5678]], @@ -264,6 +268,10 @@ class TestOCIEmbeddingConfig: raw_response=mock_response, model_response=model_response, logging_obj=mock_logging, + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, ) assert isinstance(result, EmbeddingResponse) @@ -296,6 +304,10 @@ class TestOCIEmbeddingConfig: raw_response=mock_response, model_response=model_response, logging_obj=mock_logging, + api_key=None, + request_data={}, + optional_params={}, + litellm_params={}, ) def test_model_prices_embedding_models(self): diff --git a/tests/test_litellm/llms/oci/rerank/__init__.py b/tests/test_litellm/llms/oci/rerank/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_litellm/llms/oci/test_oci_common_utils.py b/tests/test_litellm/llms/oci/test_oci_common_utils.py new file mode 100644 index 0000000000..d306d7351d --- /dev/null +++ b/tests/test_litellm/llms/oci/test_oci_common_utils.py @@ -0,0 +1,521 @@ +""" +Unit tests for litellm/llms/oci/common_utils.py. + +Covers schema utilities, signing helpers, and credential resolution paths +that require no real OCI credentials or network calls. +""" + +import pytest +from unittest.mock import MagicMock, patch + +from litellm.llms.oci.common_utils import ( + OCI_API_VERSION, + OCIError, + OCIRequestWrapper, + build_signature_string, + enrich_cohere_param_description, + get_oci_base_url, + resolve_oci_credentials, + resolve_oci_schema_anyof, + resolve_oci_schema_refs, + sanitize_oci_schema, + sha256_base64, + sign_oci_request, + sign_with_oci_signer, + validate_oci_environment, +) + +# --------------------------------------------------------------------------- +# OCI_API_VERSION +# --------------------------------------------------------------------------- + + +def test_oci_api_version_constant(): + assert OCI_API_VERSION == "20231130" + + +# --------------------------------------------------------------------------- +# sha256_base64 +# --------------------------------------------------------------------------- + + +def test_sha256_base64_known_value(): + import base64, hashlib + + data = b"hello" + expected = base64.b64encode(hashlib.sha256(data).digest()).decode() + assert sha256_base64(data) == expected + + +def test_sha256_base64_empty(): + result = sha256_base64(b"") + assert isinstance(result, str) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# build_signature_string +# --------------------------------------------------------------------------- + + +def test_build_signature_string_request_target(): + headers = {"host": "example.com", "date": "Mon, 01 Jan 2024 00:00:00 GMT"} + result = build_signature_string( + "POST", "/20231130/actions/chat", headers, ["(request-target)", "host", "date"] + ) + lines = result.split("\n") + assert lines[0] == "(request-target): post /20231130/actions/chat" + assert lines[1] == "host: example.com" + assert lines[2] == "date: Mon, 01 Jan 2024 00:00:00 GMT" + + +def test_build_signature_string_method_lowercased(): + headers = {"host": "h"} + result = build_signature_string("GET", "/path", headers, ["(request-target)"]) + assert result == "(request-target): get /path" + + +# --------------------------------------------------------------------------- +# OCIRequestWrapper.path_url +# --------------------------------------------------------------------------- + + +def test_request_wrapper_path_url_no_query(): + w = OCIRequestWrapper( + method="POST", + url="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/20231130/actions/chat", + headers={}, + body=b"", + ) + assert w.path_url == "/20231130/actions/chat" + + +def test_request_wrapper_path_url_with_query(): + w = OCIRequestWrapper( + method="GET", + url="https://example.com/path?foo=bar&baz=1", + headers={}, + body=b"", + ) + assert w.path_url == "/path?foo=bar&baz=1" + + +# --------------------------------------------------------------------------- +# resolve_oci_credentials +# --------------------------------------------------------------------------- + + +def test_resolve_credentials_from_params(): + params = { + "oci_region": "eu-frankfurt-1", + "oci_user": "user1", + "oci_fingerprint": "fp1", + "oci_tenancy": "tenant1", + "oci_key": "key_content", + "oci_compartment_id": "comp1", + } + result = resolve_oci_credentials(params) + assert result["oci_region"] == "eu-frankfurt-1" + assert result["oci_user"] == "user1" + assert result["oci_compartment_id"] == "comp1" + + +def test_resolve_credentials_env_fallback(monkeypatch): + monkeypatch.setenv("OCI_REGION", "ap-tokyo-1") + monkeypatch.setenv("OCI_USER", "env_user") + monkeypatch.setenv("OCI_COMPARTMENT_ID", "env_comp") + result = resolve_oci_credentials({}) + assert result["oci_region"] == "ap-tokyo-1" + assert result["oci_user"] == "env_user" + assert result["oci_compartment_id"] == "env_comp" + + +def test_resolve_credentials_region_default(monkeypatch): + monkeypatch.delenv("OCI_REGION", raising=False) + result = resolve_oci_credentials({}) + assert result["oci_region"] == "us-ashburn-1" + + +def test_resolve_credentials_params_override_env(monkeypatch): + monkeypatch.setenv("OCI_REGION", "ap-tokyo-1") + result = resolve_oci_credentials({"oci_region": "us-phoenix-1"}) + assert result["oci_region"] == "us-phoenix-1" + + +# --------------------------------------------------------------------------- +# get_oci_base_url +# --------------------------------------------------------------------------- + + +def test_get_oci_base_url_explicit_api_base(): + url = get_oci_base_url({}, api_base="https://custom.endpoint.com/") + assert url == "https://custom.endpoint.com" + + +@pytest.mark.parametrize( + "api_base", + [ + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat", + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/chat/", + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com/20231130/actions/embedText", + ], +) +def test_get_oci_base_url_strips_trailing_action_path(api_base): + assert ( + get_oci_base_url({}, api_base=api_base) + == "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + ) + + +def test_get_oci_base_url_from_region(): + url = get_oci_base_url({"oci_region": "eu-frankfurt-1"}) + assert url == "https://inference.generativeai.eu-frankfurt-1.oci.oraclecloud.com" + + +@pytest.mark.parametrize( + "region", + [ + "evil.com/#", + "evil.com", + "us-ashburn-1/../attacker", + "ATTACKER", + "-leading-hyphen", + "trailing-hyphen-", + "a", + "a" * 33, + "us ashburn 1", + "us_ashburn_1", + ], +) +def test_get_oci_base_url_rejects_unsafe_region(region): + with pytest.raises(OCIError, match="Invalid OCI region"): + get_oci_base_url({"oci_region": region}) + + +def test_get_oci_base_url_empty_region_falls_back_to_default(monkeypatch): + monkeypatch.delenv("OCI_REGION", raising=False) + url = get_oci_base_url({"oci_region": ""}) + assert url == "https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com" + + +@pytest.mark.parametrize( + "region", + [ + "us-ashburn-1", + "eu-frankfurt-1", + "ap-tokyo-1", + "us-chicago-1", + "us-phoenix-1", + "ap", + ], +) +def test_get_oci_base_url_accepts_valid_region(region): + url = get_oci_base_url({"oci_region": region}) + assert url == f"https://inference.generativeai.{region}.oci.oraclecloud.com" + + +# --------------------------------------------------------------------------- +# validate_oci_environment +# --------------------------------------------------------------------------- + + +def test_validate_oci_environment_sets_defaults(): + headers = {} + result = validate_oci_environment(headers, {}) + assert result["content-type"] == "application/json" + assert "user-agent" in result + + +def test_validate_oci_environment_does_not_overwrite_existing(): + headers = {"content-type": "text/plain", "user-agent": "my-agent"} + result = validate_oci_environment(headers, {}) + assert result["content-type"] == "text/plain" + assert result["user-agent"] == "my-agent" + + +# --------------------------------------------------------------------------- +# sign_with_oci_signer — error paths +# --------------------------------------------------------------------------- + + +def test_sign_with_oci_signer_none_raises(): + with pytest.raises(ValueError, match="oci_signer cannot be None"): + sign_with_oci_signer({}, {"oci_signer": None}, {}, "https://example.com") + + +def test_sign_with_oci_signer_exception_wrapped(): + bad_signer = MagicMock() + bad_signer.do_request_sign.side_effect = RuntimeError("signing failed") + with pytest.raises(OCIError, match="Failed to sign request"): + sign_with_oci_signer( + {}, {"oci_signer": bad_signer}, {"key": "val"}, "https://example.com" + ) + + +def test_sign_with_oci_signer_success(): + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_with_oci_signer( + {}, {"oci_signer": signer}, {"key": "val"}, "https://example.com" + ) + assert isinstance(body, bytes) + signer.do_request_sign.assert_called_once() + + +# --------------------------------------------------------------------------- +# sign_oci_request — routing +# --------------------------------------------------------------------------- + + +def test_sign_oci_request_routes_to_signer(): + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_oci_request( + {}, {"oci_signer": signer}, {}, "https://example.com" + ) + signer.do_request_sign.assert_called_once() + + +def test_sign_oci_request_routes_to_manual_missing_creds(): + with pytest.raises(OCIError, match="Missing required OCI credentials"): + sign_oci_request({}, {}, {}, "https://example.com") + + +# --------------------------------------------------------------------------- +# load_private_key_from_file — error paths (no real key needed) +# --------------------------------------------------------------------------- + + +def test_load_private_key_from_file_not_found(): + from litellm.llms.oci.common_utils import load_private_key_from_file + + with pytest.raises(FileNotFoundError, match="Private key file not found"): + load_private_key_from_file("/nonexistent/path/key.pem") + + +def test_load_private_key_from_file_empty(tmp_path): + from litellm.llms.oci.common_utils import load_private_key_from_file + + empty = tmp_path / "empty.pem" + empty.write_text("") + with pytest.raises(ValueError, match="Private key file is empty"): + load_private_key_from_file(str(empty)) + + +def test_load_private_key_from_file_os_error(): + from litellm.llms.oci.common_utils import load_private_key_from_file + + with patch("builtins.open", side_effect=OSError("permission denied")): + with pytest.raises(OSError, match="Failed to read private key file"): + load_private_key_from_file("/some/path/key.pem") + + +# --------------------------------------------------------------------------- +# resolve_oci_schema_refs +# --------------------------------------------------------------------------- + + +def test_resolve_schema_refs_basic(): + schema = { + "$defs": {"Foo": {"type": "string"}}, + "properties": {"x": {"$ref": "#/$defs/Foo"}}, + } + result = resolve_oci_schema_refs(schema) + assert result["properties"]["x"] == {"type": "string"} + assert "$defs" not in result + + +def test_resolve_schema_refs_external_ref_unchanged(): + schema = {"properties": {"x": {"$ref": "https://example.com/schema"}}} + result = resolve_oci_schema_refs(schema) + assert result["properties"]["x"] == {"$ref": "https://example.com/schema"} + + +def test_resolve_schema_refs_circular_breaks_cycle(): + schema = { + "$defs": {"Node": {"properties": {"child": {"$ref": "#/$defs/Node"}}}}, + "properties": {"root": {"$ref": "#/$defs/Node"}}, + } + result = resolve_oci_schema_refs(schema) + # Should not raise; circular ref replaced with {"type": "object"} + child = result["properties"]["root"]["properties"]["child"] + assert child == {"type": "object"} + + +def test_resolve_schema_refs_no_defs(): + schema = {"type": "object", "properties": {"x": {"type": "string"}}} + result = resolve_oci_schema_refs(schema) + assert result == schema + + +# --------------------------------------------------------------------------- +# resolve_oci_schema_anyof +# --------------------------------------------------------------------------- + + +def test_resolve_schema_anyof_optional_field(): + schema = {"anyOf": [{"type": "string"}, {"type": "null"}]} + result = resolve_oci_schema_anyof(schema) + assert result["type"] == "string" + assert "anyOf" not in result + + +def test_resolve_schema_anyof_all_null_returns_empty(): + schema = {"anyOf": [{"type": "null"}, {"type": "null"}]} + result = resolve_oci_schema_anyof(schema) + # No non-null branch — anyOf stays or schema unchanged + # The function only strips anyOf when there IS a non-null branch + assert "anyOf" in result + + +def test_resolve_schema_anyof_no_anyof_unchanged(): + schema = {"type": "string", "description": "A name"} + assert resolve_oci_schema_anyof(schema) == schema + + +def test_resolve_schema_anyof_nested(): + schema = {"properties": {"age": {"anyOf": [{"type": "integer"}, {"type": "null"}]}}} + result = resolve_oci_schema_anyof(schema) + assert result["properties"]["age"]["type"] == "integer" + + +# --------------------------------------------------------------------------- +# sanitize_oci_schema +# --------------------------------------------------------------------------- + + +def test_sanitize_schema_removes_title(): + schema = {"title": "MyModel", "type": "object", "properties": {}} + result = sanitize_oci_schema(schema) + assert "title" not in result + + +def test_sanitize_schema_removes_null_default(): + schema = {"type": "string", "default": None} + result = sanitize_oci_schema(schema) + assert "default" not in result + + +def test_sanitize_schema_keeps_non_null_default(): + schema = {"type": "string", "default": "hello"} + result = sanitize_oci_schema(schema) + assert result["default"] == "hello" + + +def test_sanitize_schema_type_any_becomes_object(): + schema = {"type": "any"} + result = sanitize_oci_schema(schema) + assert result["type"] == "object" + + +def test_sanitize_schema_type_list_picks_non_null(): + schema = {"type": ["string", "null"]} + result = sanitize_oci_schema(schema) + assert result["type"] == "string" + + +def test_sanitize_schema_type_list_all_null_becomes_string(): + schema = {"type": ["null"]} + result = sanitize_oci_schema(schema) + assert result["type"] == "string" + + +def test_sanitize_schema_array_gets_items(): + schema = {"type": "array"} + result = sanitize_oci_schema(schema) + assert result["items"] == {"type": "object"} + + +def test_sanitize_schema_array_keeps_existing_items(): + schema = {"type": "array", "items": {"type": "string"}} + result = sanitize_oci_schema(schema) + assert result["items"] == {"type": "string"} + + +def test_sanitize_schema_required_filters_missing_properties(): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a", "b"], # "b" not in properties + } + result = sanitize_oci_schema(schema) + assert result["required"] == ["a"] + + +def test_sanitize_schema_required_non_list_becomes_empty(): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": "a", # invalid: string instead of list + } + result = sanitize_oci_schema(schema) + assert result["required"] == [] + + +def test_sanitize_schema_list_input(): + schemas = [{"title": "A", "type": "string"}, {"title": "B", "type": "integer"}] + result = sanitize_oci_schema(schemas) + assert all("title" not in s for s in result) + + +# --------------------------------------------------------------------------- +# enrich_cohere_param_description +# --------------------------------------------------------------------------- + + +def test_enrich_description_enum(): + result = enrich_cohere_param_description("A color", {"enum": ["red", "blue"]}) + assert "Allowed values: ['red', 'blue']" in result + + +def test_enrich_description_format(): + result = enrich_cohere_param_description("A date", {"format": "date-time"}) + assert "Format: date-time" in result + + +def test_enrich_description_range_both(): + result = enrich_cohere_param_description("A number", {"minimum": 0, "maximum": 100}) + assert "Range: min=0, max=100" in result + + +def test_enrich_description_range_min_only(): + result = enrich_cohere_param_description("A number", {"minimum": 1}) + assert "Range: min=1" in result + assert "max" not in result + + +def test_enrich_description_range_max_only(): + result = enrich_cohere_param_description("", {"maximum": 10}) + assert "Range: max=10" in result + + +def test_enrich_description_pattern(): + result = enrich_cohere_param_description("An ID", {"pattern": "^[a-z]+$"}) + assert "Pattern: ^[a-z]+$" in result + + +def test_enrich_description_all_constraints(): + result = enrich_cohere_param_description( + "Val", + { + "enum": ["a"], + "format": "uuid", + "minimum": 0, + "maximum": 1, + "pattern": ".*", + }, + ) + assert "Allowed values" in result + assert "Format" in result + assert "Range" in result + assert "Pattern" in result + + +def test_enrich_description_no_constraints(): + result = enrich_cohere_param_description("Just a description", {}) + assert result == "Just a description" + + +def test_enrich_description_empty_description_no_constraints(): + result = enrich_cohere_param_description("", {}) + assert result == "" diff --git a/tests/test_litellm/llms/oci/test_oci_coverage_boost.py b/tests/test_litellm/llms/oci/test_oci_coverage_boost.py new file mode 100644 index 0000000000..0b7afa3775 --- /dev/null +++ b/tests/test_litellm/llms/oci/test_oci_coverage_boost.py @@ -0,0 +1,1152 @@ +""" +Coverage-boost tests for the OCI provider happy paths. + +Covers: + - litellm/llms/oci/common_utils.py (sign_with_manual_credentials, routing) + - litellm/llms/oci/chat/generic.py (message adaptation, tool conversion, streaming) + - litellm/llms/oci/chat/cohere.py (message adaptation, response parsing, streaming) + - litellm/llms/oci/chat/transformation.py (OCIChatConfig methods, stream wrappers) + +All tests are self-contained and require no real OCI credentials or network access. +""" + +import json +import pytest +from unittest.mock import patch, MagicMock, AsyncMock + +import httpx + +from litellm import ModelResponse +from litellm.llms.oci.chat.cohere import ( + _extract_text_content, + adapt_messages_to_cohere_standard, + handle_cohere_response, + handle_cohere_stream_chunk, +) +from litellm.llms.oci.chat.generic import ( + adapt_messages_to_generic_oci_standard, + adapt_messages_to_generic_oci_standard_tool_response, + adapt_tool_definition_to_oci_standard, + adapt_tools_to_openai_standard, + handle_generic_stream_chunk, +) +from litellm.llms.oci.chat.transformation import OCIChatConfig, get_vendor_from_model +from litellm.llms.oci.common_utils import ( + OCIError, + sign_with_manual_credentials, + sign_oci_request, + validate_oci_environment, +) +from litellm.types.llms.oci import OCIVendors, OCIToolCall + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + +_MANUAL_CREDS = { + "oci_user": "ocid1.user.oc1..xxx", + "oci_fingerprint": "aa:bb:cc:dd", + "oci_tenancy": "ocid1.tenancy.oc1..xxx", + "oci_compartment_id": "ocid1.compartment.oc1..xxx", + "oci_key": "-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----", +} + +_API_BASE = "https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/20231130/actions/chat" + +_COHERE_MODEL = "cohere.command-r-plus" +_GENERIC_MODEL = "meta.llama-3-70b-instruct" + + +# =========================================================================== +# common_utils.py — sign_with_manual_credentials happy paths +# =========================================================================== + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_inline_key( + mock_hashes, mock_padding, mock_load_key +): + """sign_with_manual_credentials succeeds with an inline oci_key string.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"fake_signature" + mock_load_key.return_value = mock_key + + result_headers, body = sign_with_manual_credentials( + {}, _MANUAL_CREDS, {"key": "val"}, _API_BASE + ) + + assert "authorization" in result_headers + assert result_headers["authorization"].startswith('Signature version="1"') + assert "rsa-sha256" in result_headers["authorization"] + assert isinstance(body, bytes) + mock_key.sign.assert_called_once() + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_file") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_key_file( + mock_hashes, mock_padding, mock_load_file +): + """sign_with_manual_credentials falls back to oci_key_file when oci_key absent.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig_from_file" + mock_load_file.return_value = mock_key + + creds = {**_MANUAL_CREDS, "oci_key_file": "/tmp/key.pem"} + creds_no_inline = {k: v for k, v in creds.items() if k != "oci_key"} + + result_headers, body = sign_with_manual_credentials( + {}, creds_no_inline, {}, _API_BASE + ) + + assert "authorization" in result_headers + mock_load_file.assert_called_once_with("/tmp/key.pem") + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_with_manual_credentials_authorization_contains_key_id( + mock_hashes, mock_padding, mock_load_key +): + """Authorization header encodes tenancy/user/fingerprint as key ID.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + result_headers, _ = sign_with_manual_credentials({}, _MANUAL_CREDS, {}, _API_BASE) + + auth = result_headers["authorization"] + assert 'keyId="ocid1.tenancy.oc1..xxx/ocid1.user.oc1..xxx/aa:bb:cc:dd"' in auth + + +def test_sign_with_manual_credentials_non_string_oci_key_raises(): + """Passing a non-string oci_key raises OCIError(400).""" + bad_creds = {**_MANUAL_CREDS, "oci_key": 12345} + with pytest.raises(OCIError) as exc_info: + sign_with_manual_credentials({}, bad_creds, {}, _API_BASE) + assert exc_info.value.status_code == 400 + assert "oci_key must be a string" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# common_utils.py — sign_oci_request routing +# --------------------------------------------------------------------------- + + +def test_sign_oci_request_routes_to_signer_when_present(): + """sign_oci_request delegates to sign_with_oci_signer when oci_signer is set.""" + signer = MagicMock() + signer.do_request_sign.return_value = None + headers, body = sign_oci_request({}, {"oci_signer": signer}, {"data": 1}, _API_BASE) + signer.do_request_sign.assert_called_once() + assert isinstance(body, bytes) + + +@patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) +@patch("litellm.llms.oci.common_utils.load_private_key_from_str") +@patch("litellm.llms.oci.common_utils.padding") +@patch("litellm.llms.oci.common_utils.hashes") +def test_sign_oci_request_routes_to_manual_when_no_signer( + mock_hashes, mock_padding, mock_load_key +): + """sign_oci_request delegates to sign_with_manual_credentials when oci_signer absent.""" + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + headers, body = sign_oci_request({}, _MANUAL_CREDS, {}, _API_BASE) + assert "authorization" in headers + + +# --------------------------------------------------------------------------- +# common_utils.py — _require_cryptography happy path +# --------------------------------------------------------------------------- + + +def test_require_cryptography_available_does_not_raise(): + """_require_cryptography() should not raise when the package is importable.""" + from litellm.llms.oci.common_utils import _require_cryptography + + with patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True): + _require_cryptography() # must not raise + + +# =========================================================================== +# generic.py — adapt_messages_to_generic_oci_standard +# =========================================================================== + + +def test_adapt_generic_user_message_string_content(): + messages = [{"role": "user", "content": "Hello!"}] + result = adapt_messages_to_generic_oci_standard(messages) + assert len(result) == 1 + assert result[0].role == "USER" + assert result[0].content[0].text == "Hello!" + + +def test_adapt_generic_assistant_message(): + messages = [{"role": "assistant", "content": "Hi there!"}] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "ASSISTANT" + assert result[0].content[0].text == "Hi there!" + + +def test_adapt_generic_system_message(): + messages = [{"role": "system", "content": "You are a helpful assistant."}] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "SYSTEM" + assert result[0].content[0].text == "You are a helpful assistant." + + +def test_adapt_generic_tool_message(): + messages = [ + { + "role": "tool", + "tool_call_id": "call_abc123", + "content": "42 degrees", + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "TOOL" + assert result[0].toolCallId == "call_abc123" + assert result[0].content[0].text == "42 degrees" + + +def test_adapt_generic_assistant_tool_call_message(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_xyz", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Rome"}', + }, + } + ], + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert result[0].role == "ASSISTANT" + assert result[0].toolCalls is not None + assert len(result[0].toolCalls) == 1 + tc = result[0].toolCalls[0] + assert tc.name == "get_weather" + assert tc.arguments == '{"city": "Rome"}' + + +def test_adapt_generic_multipart_content(): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Look at this:"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.png"}, + }, + ], + } + ] + result = adapt_messages_to_generic_oci_standard(messages) + assert len(result[0].content) == 2 + assert result[0].content[0].text == "Look at this:" + assert result[0].content[1].imageUrl.url == "https://example.com/img.png" + + +# --------------------------------------------------------------------------- +# generic.py — adapt_messages_to_generic_oci_standard_tool_response +# --------------------------------------------------------------------------- + + +def test_adapt_generic_tool_response_direct(): + result = adapt_messages_to_generic_oci_standard_tool_response( + "tool", "call_999", "The answer is 42" + ) + assert result.role == "TOOL" + assert result.toolCallId == "call_999" + assert result.content[0].text == "The answer is 42" + + +# --------------------------------------------------------------------------- +# generic.py — adapt_tool_definition_to_oci_standard +# --------------------------------------------------------------------------- + + +def test_adapt_tool_definition_basic(): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Retrieve current weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + result = adapt_tool_definition_to_oci_standard(tools, OCIVendors.GENERIC) + assert len(result) == 1 + tool_def = result[0] + assert tool_def.name == "get_weather" + assert tool_def.type == "FUNCTION" + assert tool_def.parameters is not None + + +def test_adapt_tool_definition_resolves_refs(): + """$ref/$defs schemas are inlined before being sent to OCI.""" + tools = [ + { + "type": "function", + "function": { + "name": "do_thing", + "parameters": { + "$defs": {"Loc": {"type": "string"}}, + "type": "object", + "properties": {"location": {"$ref": "#/$defs/Loc"}}, + }, + }, + } + ] + result = adapt_tool_definition_to_oci_standard(tools, OCIVendors.GENERIC) + props = result[0].parameters["properties"] + assert props["location"] == {"type": "string"} + + +# --------------------------------------------------------------------------- +# generic.py — adapt_tools_to_openai_standard +# --------------------------------------------------------------------------- + + +def test_adapt_tools_to_openai_standard(): + oci_tool = OCIToolCall( + id="call_abc", + type="FUNCTION", + name="search", + arguments='{"query": "hello"}', + ) + result = adapt_tools_to_openai_standard([oci_tool]) + assert len(result) == 1 + assert result[0].id == "call_abc" + assert result[0].type == "function" + assert result[0].function["name"] == "search" + + +def test_adapt_tools_to_openai_standard_generates_id_when_absent(): + oci_tool = OCIToolCall( + id=None, + type="FUNCTION", + name="lookup", + arguments="{}", + ) + result = adapt_tools_to_openai_standard([oci_tool]) + assert result[0].id.startswith("call_") + + +# --------------------------------------------------------------------------- +# generic.py — handle_generic_stream_chunk +# --------------------------------------------------------------------------- + + +def test_handle_generic_stream_chunk_text_content(): + chunk = { + "message": { + "content": [{"type": "TEXT", "text": "Hello from OCI"}], + "role": "ASSISTANT", + }, + "finishReason": None, + "index": 0, + } + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.content == "Hello from OCI" + assert result.choices[0].finish_reason is None + + +def test_handle_generic_stream_chunk_complete_finish_reason(): + chunk = {"finishReason": "COMPLETE", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "stop" + + +def test_handle_generic_stream_chunk_max_tokens_finish_reason(): + chunk = {"finishReason": "MAX_TOKENS", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "length" + + +def test_handle_generic_stream_chunk_tool_calls_finish_reason(): + chunk = {"finishReason": "TOOL_CALLS", "index": 0} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + + +def test_handle_generic_stream_chunk_no_message(): + """Chunks without a message key should still parse without error.""" + chunk = {"finishReason": "COMPLETE", "index": 1} + result = handle_generic_stream_chunk(chunk) + assert result.choices[0].delta.content is None + assert result.choices[0].finish_reason == "stop" + + +# =========================================================================== +# cohere.py — _extract_text_content +# =========================================================================== + + +def test_extract_text_content_none(): + assert _extract_text_content(None) == "" + + +def test_extract_text_content_string(): + assert _extract_text_content("hello") == "hello" + + +def test_extract_text_content_list(): + content = [ + {"type": "text", "text": "foo"}, + {"type": "text", "text": "bar"}, + ] + assert _extract_text_content(content) == "foobar" + + +def test_extract_text_content_list_skips_non_text(): + content = [ + {"type": "image_url", "url": "https://x.com/img.png"}, + {"type": "text", "text": "only this"}, + ] + assert _extract_text_content(content) == "only this" + + +def test_extract_text_content_non_string_non_list(): + assert _extract_text_content(42) == "42" + + +# =========================================================================== +# cohere.py — adapt_messages_to_cohere_standard +# =========================================================================== + + +def test_adapt_cohere_user_in_history(): + messages = [ + {"role": "user", "content": "first question"}, + {"role": "user", "content": "current question"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 1 + assert history[0].role == "USER" + assert history[0].message == "first question" + + +def test_adapt_cohere_assistant_in_history(): + messages = [ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "answer"}, + {"role": "user", "content": "follow-up"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 2 + chatbot_msg = history[1] + assert chatbot_msg.role == "CHATBOT" + assert chatbot_msg.message == "answer" + + +def test_adapt_cohere_assistant_with_tool_calls_in_history(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "calc", "arguments": '{"x": 1}'}, + } + ], + }, + {"role": "user", "content": "thanks"}, + ] + history = adapt_messages_to_cohere_standard(messages) + assert len(history) == 1 + assert history[0].role == "CHATBOT" + assert history[0].toolCalls is not None + assert history[0].toolCalls[0].name == "calc" + + +def test_adapt_cohere_tool_result_in_history(): + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "calc", "arguments": '{"x": 1}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "result: 42", + }, + {"role": "user", "content": "ok"}, + ] + history = adapt_messages_to_cohere_standard(messages) + tool_msg = next(m for m in history if m.role == "TOOL") + assert tool_msg.toolResults[0].call.name == "calc" + assert tool_msg.toolResults[0].outputs[0]["output"] == "result: 42" + + +# =========================================================================== +# cohere.py — handle_cohere_response +# =========================================================================== + + +_COHERE_RESPONSE_JSON = { + "modelId": "cohere.command-r-plus", + "modelVersion": "1.0", + "chatResponse": { + "apiFormat": "COHERE", + "text": "Hello from Cohere!", + "finishReason": "COMPLETE", + "usage": { + "promptTokens": 10, + "completionTokens": 5, + "totalTokens": 15, + }, + }, +} + + +_COHERE_RAW_RESPONSE = httpx.Response(200, request=httpx.Request("POST", "https://oci")) + + +def test_handle_cohere_response_complete(): + model_response = ModelResponse() + result = handle_cohere_response( + _COHERE_RESPONSE_JSON, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].message["content"] == "Hello from Cohere!" + assert result.usage.prompt_tokens == 10 + + +def test_handle_cohere_response_max_tokens(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + **_COHERE_RESPONSE_JSON["chatResponse"], + "finishReason": "MAX_TOKENS", + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "length" + + +def test_handle_cohere_response_tool_call(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + **_COHERE_RESPONSE_JSON["chatResponse"], + "finishReason": "TOOL_CALL", + "toolCalls": [{"name": "get_time", "parameters": {"tz": "UTC"}}], + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.choices[0].finish_reason == "tool_calls" + tool_calls = result.choices[0].message["tool_calls"] + assert tool_calls is not None + assert tool_calls[0]["function"]["name"] == "get_time" + + +def test_handle_cohere_response_missing_usage(): + resp = { + **_COHERE_RESPONSE_JSON, + "chatResponse": { + k: v + for k, v in _COHERE_RESPONSE_JSON["chatResponse"].items() + if k != "usage" + }, + } + model_response = ModelResponse() + result = handle_cohere_response( + resp, _COHERE_MODEL, model_response, _COHERE_RAW_RESPONSE + ) + assert result.usage.prompt_tokens == 0 + assert result.usage.completion_tokens == 0 + assert result.usage.total_tokens == 0 + + +def test_handle_cohere_response_malformed_raises_oci_error(): + bad_json = {"chatResponse": {"apiFormat": "COHERE"}} + raw = httpx.Response(502, request=httpx.Request("POST", "https://oci")) + model_response = ModelResponse() + with pytest.raises(OCIError) as exc_info: + handle_cohere_response(bad_json, _COHERE_MODEL, model_response, raw) + assert exc_info.value.status_code == 502 + + +# =========================================================================== +# cohere.py — handle_cohere_stream_chunk +# =========================================================================== + + +def test_handle_cohere_stream_chunk_text(): + chunk = {"apiFormat": "COHERE", "text": "streaming text", "finishReason": None} + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "streaming text" + assert result.choices[0].finish_reason is None + + +def test_handle_cohere_stream_chunk_complete(): + # Real OCI Cohere terminal events carry the full response in `text` plus a + # populated `chatHistory`; the parser must drop that text to avoid doubling + # — but only when prior chunks already emitted the text as incremental + # deltas (signalled by ``prior_text_emitted=True``). + chunk = { + "apiFormat": "COHERE", + "text": "How can I help you today?", + "finishReason": "COMPLETE", + "chatHistory": [ + {"role": "USER", "message": "Hello!"}, + {"role": "CHATBOT", "message": "How can I help you today?"}, + ], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].finish_reason == "stop" + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_max_tokens(): + chunk = { + "apiFormat": "COHERE", + "text": "truncated full response", + "finishReason": "MAX_TOKENS", + "chatHistory": [{"role": "CHATBOT", "message": "truncated full response"}], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].finish_reason == "length" + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_tool_call(): + chunk = { + "apiFormat": "COHERE", + "text": "", + "finishReason": "TOOL_CALL", + "chatHistory": [{"role": "CHATBOT", "message": ""}], + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].finish_reason == "tool_calls" + assert not result.choices[0].delta.content + + +def test_handle_cohere_stream_chunk_terminal_drops_full_response_text(): + """Regression for double-output on cohere.command-* streaming. + + OCI's terminal SSE event re-sends the full assembled response in `text` + alongside a populated `chatHistory`. That text must be dropped — otherwise + it gets concatenated onto the already-streamed incremental deltas. The + caller signals "prior deltas already emitted text" via + ``prior_text_emitted=True``. + """ + chunk = { + "apiFormat": "COHERE", + "text": "How can I help you today?", + "finishReason": "COMPLETE", + "chatHistory": [ + {"role": "USER", "message": "Hello!"}, + {"role": "CHATBOT", "message": "How can I help you today?"}, + ], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=True) + assert result.choices[0].delta.content is None + + +def test_handle_cohere_stream_chunk_single_event_stream_preserves_text(): + """Degenerate single-event stream: the terminal chunk carries the only copy + of the response text. Without prior text deltas, suppressing here would + discard the response entirely — so the text must pass through.""" + chunk = { + "apiFormat": "COHERE", + "text": "Short answer.", + "finishReason": "COMPLETE", + "chatHistory": [{"role": "CHATBOT", "message": "Short answer."}], + } + result = handle_cohere_stream_chunk(chunk, prior_text_emitted=False) + assert result.choices[0].delta.content == "Short answer." + assert result.choices[0].finish_reason == "stop" + + +def test_handle_cohere_stream_chunk_incremental_passes_text_through(): + """Non-terminal chunks (no chatHistory) must emit their incremental text.""" + chunk = { + "apiFormat": "COHERE", + "text": "How can I ", + "finishReason": None, + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "How can I " + assert result.choices[0].finish_reason is None + + +def test_handle_cohere_stream_chunk_finish_reason_without_chathistory_keeps_text(): + """`finishReason` alone (no `chatHistory`) must NOT trigger the drop — + `chatHistory` is the discriminator for the consolidated terminal event.""" + chunk = { + "apiFormat": "COHERE", + "text": "tail delta", + "finishReason": "COMPLETE", + } + result = handle_cohere_stream_chunk(chunk) + assert result.choices[0].delta.content == "tail delta" + assert result.choices[0].finish_reason == "stop" + + +# =========================================================================== +# transformation.py — get_vendor_from_model +# =========================================================================== + + +def test_get_vendor_cohere(): + assert get_vendor_from_model("cohere.command-r-plus") == OCIVendors.COHERE + + +def test_get_vendor_generic_llama(): + assert get_vendor_from_model("meta.llama-3-70b-instruct") == OCIVendors.GENERIC + + +def test_get_vendor_generic_xai(): + assert get_vendor_from_model("xai.grok-4") == OCIVendors.GENERIC + + +def test_get_vendor_generic_google(): + assert get_vendor_from_model("google.gemini-2-flash") == OCIVendors.GENERIC + + +# =========================================================================== +# transformation.py — OCIChatConfig methods +# =========================================================================== + + +class TestOCIChatConfigGetCompleteUrl: + def test_returns_chat_endpoint_from_region(self): + config = OCIChatConfig() + url = config.get_complete_url( + api_base=None, + api_key=None, + model=_GENERIC_MODEL, + optional_params={"oci_region": "eu-frankfurt-1"}, + litellm_params={}, + ) + assert url == ( + "https://inference.generativeai.eu-frankfurt-1.oci.oraclecloud.com" + "/20231130/actions/chat" + ) + + def test_respects_explicit_api_base(self): + config = OCIChatConfig() + url = config.get_complete_url( + api_base="https://custom.endpoint.com/", + api_key=None, + model=_GENERIC_MODEL, + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.endpoint.com/20231130/actions/chat" + + def test_full_chat_url_is_not_doubled(self): + config = OCIChatConfig() + full_url = ( + "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + "/20231130/actions/chat" + ) + url = config.get_complete_url( + api_base=full_url, + api_key=None, + model=_GENERIC_MODEL, + optional_params={}, + litellm_params={}, + ) + assert url == full_url + + +class TestOCIChatConfigGetErrorClass: + def test_returns_oci_error(self): + config = OCIChatConfig() + err = config.get_error_class("boom", 503, {}) + assert isinstance(err, OCIError) + assert err.status_code == 503 + + +class TestOCIChatConfigSignRequest: + @patch("litellm.llms.oci.common_utils._CRYPTOGRAPHY_AVAILABLE", True) + @patch("litellm.llms.oci.common_utils.load_private_key_from_str") + @patch("litellm.llms.oci.common_utils.padding") + @patch("litellm.llms.oci.common_utils.hashes") + def test_sign_request_delegates(self, mock_hashes, mock_padding, mock_load_key): + mock_key = MagicMock() + mock_key.sign.return_value = b"sig" + mock_load_key.return_value = mock_key + + config = OCIChatConfig() + headers, body = config.sign_request( + headers={}, + optional_params=_MANUAL_CREDS, + request_data={"hello": "world"}, + api_base=_API_BASE, + ) + assert "authorization" in headers + assert isinstance(body, bytes) + + +class TestOCIChatConfigValidateEnvironment: + def test_with_signer_skips_credential_check(self): + """If oci_signer is provided, validate_environment must NOT raise.""" + config = OCIChatConfig() + signer = MagicMock() + result = config.validate_environment( + headers={}, + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + optional_params={"oci_signer": signer}, + litellm_params={}, + ) + assert result["content-type"] == "application/json" + + def test_raises_when_messages_empty(self): + config = OCIChatConfig() + with pytest.raises(OCIError) as exc_info: + config.validate_environment( + headers={}, + model=_GENERIC_MODEL, + messages=[], + optional_params={"oci_signer": MagicMock()}, + litellm_params={}, + ) + assert exc_info.value.status_code == 400 + + +class TestOCIChatConfigGetOptionalParams: + def _config(self): + return OCIChatConfig() + + def test_cohere_maps_stop_to_stop_sequences(self): + config = self._config() + result = config._get_optional_params(OCIVendors.COHERE, {"stop": ["END"]}) + assert "stopSequences" in result + assert result["stopSequences"] == ["END"] + + def test_generic_maps_max_tokens(self): + config = self._config() + result = config._get_optional_params(OCIVendors.GENERIC, {"max_tokens": 512}) + assert result["maxTokens"] == 512 + + def test_tool_choice_string_auto_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "auto"} + ) + assert result["toolChoice"] == {"type": "AUTO"} + + def test_tool_choice_string_none_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "none"} + ) + assert result["toolChoice"] == {"type": "NONE"} + + def test_tool_choice_string_required_converted_to_dict(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": "required"} + ) + assert result["toolChoice"] == {"type": "REQUIRED"} + + def test_tool_choice_openai_function_dict_converted_to_oci_form(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, + { + "tool_choice": { + "type": "function", + "function": {"name": "my_func"}, + } + }, + ) + assert result["toolChoice"] == {"type": "FUNCTION", "name": "my_func"} + + def test_tool_choice_flat_function_dict_uppercased(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, + {"tool_choice": {"type": "function", "name": "my_func"}}, + ) + assert result["toolChoice"] == {"type": "FUNCTION", "name": "my_func"} + + def test_tool_choice_dict_auto_uppercased(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"tool_choice": {"type": "auto"}} + ) + assert result["toolChoice"] == {"type": "AUTO"} + + def test_response_format_json_generic(self): + config = self._config() + result = config._get_optional_params( + OCIVendors.GENERIC, {"response_format": {"type": "json_object"}} + ) + assert result["responseFormat"]["type"] == "JSON_OBJECT" + + def test_tools_adapted_for_cohere(self): + config = self._config() + tools = [ + { + "type": "function", + "function": { + "name": "echo", + "description": "echo", + "parameters": { + "type": "object", + "properties": {"msg": {"type": "string"}}, + "required": ["msg"], + }, + }, + } + ] + result = config._get_optional_params(OCIVendors.COHERE, {"tools": tools}) + # tools should be CohereTool objects + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "echo" + + def test_tools_adapted_for_generic(self): + config = self._config() + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "search the web", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + }, + }, + } + ] + result = config._get_optional_params(OCIVendors.GENERIC, {"tools": tools}) + assert len(result["tools"]) == 1 + assert result["tools"][0].name == "search" + + +class TestOCIChatConfigTransformRequest: + _base_params = {**_MANUAL_CREDS} + + def test_generic_model_transform(self): + config = OCIChatConfig() + result = config.transform_request( + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hello"}], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + assert result["compartmentId"] == _MANUAL_CREDS["oci_compartment_id"] + chat_req = result["chatRequest"] + assert chat_req["apiFormat"] == "GENERIC" + + def test_cohere_model_transform(self): + config = OCIChatConfig() + result = config.transform_request( + model=_COHERE_MODEL, + messages=[{"role": "user", "content": "tell me a joke"}], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + chat_req = result["chatRequest"] + assert chat_req["apiFormat"] == "COHERE" + assert chat_req["message"] == "tell me a joke" + + def test_cohere_model_with_system_preamble(self): + config = OCIChatConfig() + result = config.transform_request( + model=_COHERE_MODEL, + messages=[ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "hi"}, + ], + optional_params=self._base_params, + litellm_params={}, + headers={}, + ) + assert result["chatRequest"]["preambleOverride"] == "You are helpful." + + def test_raises_without_compartment_id(self): + config = OCIChatConfig() + params = {k: v for k, v in _MANUAL_CREDS.items() if k != "oci_compartment_id"} + with pytest.raises(OCIError) as exc_info: + config.transform_request( + model=_GENERIC_MODEL, + messages=[{"role": "user", "content": "hi"}], + optional_params=params, + litellm_params={}, + headers={}, + ) + assert exc_info.value.status_code == 400 + assert "oci_compartment_id" in str(exc_info.value) + + +# =========================================================================== +# transformation.py — OCIStreamWrapper.chunk_creator +# =========================================================================== + + +class TestOCIStreamWrapperChunkCreator: + def _make_wrapper(self, model: str) -> "OCIStreamWrapper": + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + return OCIStreamWrapper( + completion_stream=iter([]), + model=model, + custom_llm_provider="oci", + logging_obj=MagicMock(), + ) + + def test_cohere_chunk_dispatched_correctly(self): + wrapper = self._make_wrapper(_COHERE_MODEL) + payload = json.dumps( + {"apiFormat": "COHERE", "text": "hi", "finishReason": None} + ) + result = wrapper.chunk_creator(f"data:{payload}") + assert result.choices[0].delta.content == "hi" + + def test_generic_chunk_dispatched_correctly(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + payload = json.dumps( + { + "finishReason": "COMPLETE", + "index": 0, + } + ) + result = wrapper.chunk_creator(f"data:{payload}") + assert result.choices[0].finish_reason == "stop" + + def test_raises_on_non_data_prefix(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + with pytest.raises(ValueError, match="does not start with 'data:'"): + wrapper.chunk_creator("event: done") + + def test_raises_on_non_string_chunk(self): + wrapper = self._make_wrapper(_GENERIC_MODEL) + with pytest.raises(ValueError, match="not a string"): + wrapper.chunk_creator({"bad": "type"}) + + def test_empty_string_content_does_not_mark_text_emitted(self): + # An intermediate Cohere chunk carrying `text=""` must not flip the + # _cohere_text_emitted flag — otherwise a subsequent terminal + # consolidation chunk would have its real text suppressed as a + # "duplicate" and the response would be lost. + wrapper = self._make_wrapper(_COHERE_MODEL) + empty_payload = json.dumps( + {"apiFormat": "COHERE", "text": "", "finishReason": None} + ) + wrapper.chunk_creator(f"data:{empty_payload}") + assert wrapper._cohere_text_emitted is False + + terminal_payload = json.dumps( + { + "apiFormat": "COHERE", + "text": "Hello world", + "finishReason": "COMPLETE", + "chatHistory": [{"role": "CHATBOT", "message": "Hello world"}], + } + ) + result = wrapper.chunk_creator(f"data:{terminal_payload}") + assert result.choices[0].delta.content == "Hello world" + + +# =========================================================================== +# transformation.py — get_sync_custom_stream_wrapper +# =========================================================================== + + +def test_get_sync_custom_stream_wrapper_returns_wrapper(): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + config = OCIChatConfig() + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.iter_text.return_value = iter( + ['data:{"finishReason":"COMPLETE","index":0}'] + ) + + mock_client = MagicMock() + mock_client.post.return_value = mock_response + + wrapper = config.get_sync_custom_stream_wrapper( + model=_GENERIC_MODEL, + custom_llm_provider="oci", + logging_obj=MagicMock(), + api_base=_API_BASE, + headers={"authorization": "Signature ..."}, + data={"chatRequest": {}}, + messages=[{"role": "user", "content": "hi"}], + client=mock_client, + signed_json_body=b'{"chatRequest":{}}', + ) + + assert isinstance(wrapper, OCIStreamWrapper) + mock_client.post.assert_called_once() + + +@pytest.mark.asyncio +async def test_get_async_custom_stream_wrapper_returns_wrapper(): + from litellm.llms.oci.chat.transformation import OCIStreamWrapper + + config = OCIChatConfig() + + async def _fake_aiter_text(): + yield 'data:{"finishReason":"COMPLETE","index":0}' + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.aiter_text = _fake_aiter_text + + mock_client = AsyncMock() + mock_client.post = AsyncMock(return_value=mock_response) + + wrapper = await config.get_async_custom_stream_wrapper( + model=_GENERIC_MODEL, + custom_llm_provider="oci", + logging_obj=MagicMock(), + api_base=_API_BASE, + headers={"authorization": "Signature ..."}, + data={"chatRequest": {}}, + messages=[{"role": "user", "content": "hi"}], + client=mock_client, + signed_json_body=b'{"chatRequest":{}}', + ) + + assert isinstance(wrapper, OCIStreamWrapper)