From 36bedc69ff3b42f314c7dd4b65d3a36e1c2015d3 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 18 Sep 2025 23:51:35 +0530 Subject: [PATCH] Add TwelveLabs marengo model (#14674) --- litellm/constants.py | 5 +- litellm/llms/bedrock/embed/embedding.py | 60 +++++--- .../twelvelabs_marengo_transformation.py | 131 ++++++++++++++++++ litellm/types/llms/bedrock.py | 29 ++++ model_prices_and_context_window.json | 12 ++ .../bedrock/embed/test_bedrock_embedding.py | 9 ++ 6 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py diff --git a/litellm/constants.py b/litellm/constants.py index 3abab5dbd0..077059b7a3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -822,6 +822,7 @@ bedrock_embedding_models: set = set( "amazon.titan-embed-text-v1", "cohere.embed-english-v3", "cohere.embed-multilingual-v3", + "twelvelabs.marengo-embed-2-7-v1:0", ] ) @@ -1065,4 +1066,6 @@ SENTRY_PII_DENYLIST = [ ] # CoroutineChecker cache configuration -COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)) \ No newline at end of file +COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int( + os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000) +) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 0824905f51..67ece820b1 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -4,8 +4,8 @@ Handles embedding calls to Bedrock's `/invoke` endpoint import copy import json -from typing import Any, Callable, List, Optional, Tuple, Union import urllib.parse +from typing import Any, Callable, List, Optional, Tuple, Union import httpx @@ -18,7 +18,11 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, ) from litellm.secret_managers.main import get_secret -from litellm.types.llms.bedrock import AmazonEmbeddingRequest, CohereEmbeddingRequest +from litellm.types.llms.bedrock import ( + AmazonEmbeddingRequest, + CohereEmbeddingRequest, + TwelveLabsMarengoEmbeddingRequest, +) from litellm.types.utils import EmbeddingResponse from ..base_aws_llm import BaseAWSLLM @@ -29,6 +33,7 @@ from .amazon_titan_multimodal_transformation import ( ) from .amazon_titan_v2_transformation import AmazonTitanV2Config from .cohere_transformation import BedrockCohereEmbeddingConfig +from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig class BedrockEmbedding(BaseAWSLLM): @@ -164,16 +169,16 @@ class BedrockEmbedding(BaseAWSLLM): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} - + prepped = self.get_request_headers( - credentials=credentials, - aws_region_name=aws_region_name, - extra_headers=extra_headers, - endpoint_url=endpoint_url, - data=json.dumps(data), - headers=headers, - api_key=api_key - ) + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=json.dumps(data), + headers=headers, + api_key=api_key, + ) ## LOGGING logging_obj.pre_call( @@ -248,16 +253,16 @@ class BedrockEmbedding(BaseAWSLLM): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} - + prepped = self.get_request_headers( - credentials=credentials, - aws_region_name=aws_region_name, - extra_headers=extra_headers, - endpoint_url=endpoint_url, - data=json.dumps(data), - headers=headers, - api_key=api_key, - ) + credentials=credentials, + aws_region_name=aws_region_name, + extra_headers=extra_headers, + endpoint_url=endpoint_url, + data=json.dumps(data), + headers=headers, + api_key=api_key, + ) ## LOGGING logging_obj.pre_call( @@ -336,7 +341,7 @@ class BedrockEmbedding(BaseAWSLLM): ### TRANSFORMATION ### unencoded_model_id = ( optional_params.pop("model_id", None) or model - ) # default to model if not passed + ) # default to model if not passed modelId = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( optional_params=optional_params, @@ -394,6 +399,17 @@ class BedrockEmbedding(BaseAWSLLM): ) ) batch_data.append(transformed_request) + elif provider == "twelvelabs" and model in [ + "twelvelabs.marengo-embed-2-7-v1:0", + ]: + batch_data = [] + for i in input: + twelvelabs_request: ( + TwelveLabsMarengoEmbeddingRequest + ) = TwelveLabsMarengoEmbeddingConfig()._transform_request( + input=i, inference_params=inference_params + ) + batch_data.append(twelvelabs_request) ### SET RUNTIME ENDPOINT ### endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( @@ -445,7 +461,7 @@ class BedrockEmbedding(BaseAWSLLM): headers = {"Content-Type": "application/json"} if extra_headers is not None: headers = {"Content-Type": "application/json", **extra_headers} - + prepped = self.get_request_headers( credentials=credentials, aws_region_name=aws_region_name, diff --git a/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py b/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py new file mode 100644 index 0000000000..ffa1ed940e --- /dev/null +++ b/litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py @@ -0,0 +1,131 @@ +""" +Transformation logic from OpenAI /v1/embeddings format to Bedrock TwelveLabs Marengo /invoke format. + +Why separate file? Make it easy to see how transformation works + +Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html +""" + +from typing import List + +from litellm.types.llms.bedrock import ( + TwelveLabsMarengoEmbeddingRequest, +) +from litellm.types.utils import Embedding, EmbeddingResponse, Usage +from litellm.utils import get_base64_str, is_base64_encoded + + +class TwelveLabsMarengoEmbeddingConfig: + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html + + Supports text and image inputs for Phase 1. + Video and audio support will be added in Phase 2. + """ + + def __init__(self) -> None: + pass + + def get_supported_openai_params(self) -> List[str]: + return ["encoding_format", "textTruncate", "embeddingOption"] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for k, v in non_default_params.items(): + if k == "encoding_format": + # TwelveLabs doesn't have encoding_format, but we can map it to embeddingOption + if v == "float": + optional_params["embeddingOption"] = ["visual-text", "visual-image"] + elif k == "textTruncate": + optional_params["textTruncate"] = v + elif k == "embeddingOption": + optional_params["embeddingOption"] = v + return optional_params + + def _transform_request( + self, input: str, inference_params: dict + ) -> TwelveLabsMarengoEmbeddingRequest: + """ + Transform OpenAI-style input to TwelveLabs Marengo format. + Phase 1: Supports text and image inputs only. + """ + # Check if input is base64 encoded image + is_encoded = is_base64_encoded(input) + + if is_encoded: + # Image input + b64_str = get_base64_str(input) + transformed_request = TwelveLabsMarengoEmbeddingRequest( + inputType="image", mediaSource={"base64String": b64_str} + ) + else: + # Text input + transformed_request = TwelveLabsMarengoEmbeddingRequest( + inputType="text", inputText=input + ) + + # Set default textTruncate if not specified + if "textTruncate" not in inference_params: + transformed_request["textTruncate"] = "end" + + # Set default embedding options for Phase 1 (text and image) + if "embeddingOption" not in inference_params: + if is_encoded: + # For images, return both visual-text and visual-image embeddings + transformed_request["embeddingOption"] = ["visual-text", "visual-image"] + else: + # For text, return visual-text embedding + transformed_request["embeddingOption"] = ["visual-text"] + + # Apply any additional inference parameters + for k, v in inference_params.items(): + if k not in [ + "inputType", + "inputText", + "mediaSource", + ]: # Don't override core fields + transformed_request[k] = v # type: ignore + + return transformed_request + + def _transform_response( + self, response_list: List[dict], model: str + ) -> EmbeddingResponse: + """ + Transform TwelveLabs response to OpenAI format. + Handles multiple embedding types in the response. + """ + embeddings: List[Embedding] = [] + total_tokens = 0 + + for response in response_list: + if "embedding" in response: + # Single embedding response + embedding = Embedding( + embedding=response["embedding"], + index=len(embeddings), + object="embedding", + ) + embeddings.append(embedding) + + # Estimate token count (rough approximation) + if "inputTextTokenCount" in response: + total_tokens += response["inputTextTokenCount"] + else: + # Rough estimate: 1 token per 4 characters for text + total_tokens += len(response.get("inputText", "")) // 4 + elif "embeddings" in response: + # Multiple embeddings response (from video/audio) + for i, emb in enumerate(response["embeddings"]): + embedding = Embedding( + embedding=emb["embedding"], + index=len(embeddings), + object="embedding", + ) + embeddings.append(embedding) + total_tokens += len(emb["embedding"]) // 4 # Rough estimate + + usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens) + + return EmbeddingResponse(data=embeddings, model=model, usage=usage) diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index d225008c20..e69daf632a 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -364,6 +364,35 @@ class AmazonTitanMultimodalEmbeddingResponse(TypedDict): message: str # Specifies any errors that occur during generation. +# TwelveLabs Marengo Embed 2.7 types +TWELVELABS_EMBEDDING_INPUT_TYPES = Literal["text", "image", "video", "audio"] +TWELVELABS_EMBEDDING_OPTIONS = Literal["visual-text", "visual-image", "audio"] + + +class TwelveLabsMediaSource(TypedDict, total=False): + base64String: str + s3Location: dict # {"uri": str, "bucketOwner": str} + + +class TwelveLabsMarengoEmbeddingRequest(TypedDict, total=False): + inputType: Required[TWELVELABS_EMBEDDING_INPUT_TYPES] + inputText: str + mediaSource: TwelveLabsMediaSource + textTruncate: Literal["end", "none"] + startSec: float + lengthSec: float + useFixedLengthSec: float + minClipSec: int + embeddingOption: List[TWELVELABS_EMBEDDING_OPTIONS] + + +class TwelveLabsMarengoEmbeddingResponse(TypedDict): + embedding: List[float] + embeddingOption: TWELVELABS_EMBEDDING_OPTIONS + startSec: float + endSec: float + + AmazonEmbeddingRequest = Union[ AmazonTitanMultimodalEmbeddingRequest, AmazonTitanV2EmbeddingRequest, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5e5bccb81e..da2700f7c2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -296,6 +296,18 @@ "output_cost_per_token": 0.0, "output_vector_size": 1024 }, + "twelvelabs.marengo-embed-2-7-v1:0": { + "input_cost_per_token": 7e-05, + "litellm_provider": "bedrock", + "max_input_tokens": 77, + "max_tokens": 77, + "mode": "embedding", + "output_cost_per_token": 0.0, + "output_vector_size": 1024, + "supports_embedding_image_input": true, + "supports_image_input": true, + "supports_multimodal_embedding": true + }, "amazon.titan-text-express-v1": { "input_cost_per_token": 1.3e-06, "litellm_provider": "bedrock", diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index aec0b5fc6c..c8d5bb860a 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -19,6 +19,13 @@ cohere_embedding_response = { "inputTextTokenCount": 10 } +twelvelabs_embedding_response = { + "embedding": [0.1, 0.2, 0.3], + "embeddingOption": "visual-text", + "startSec": 0.0, + "endSec": 1.0 +} + # Test data test_input = "Hello world from litellm" test_image_base64 = "data:image/png,test_image_base64_data" @@ -32,6 +39,8 @@ test_image_base64 = "data:image/png,test_image_base64_data" ("bedrock/amazon.titan-embed-image-v1", "image", titan_embedding_response), ("bedrock/cohere.embed-english-v3", "text", cohere_embedding_response), ("bedrock/cohere.embed-multilingual-v3", "text", cohere_embedding_response), + ("bedrock/twelvelabs.marengo-embed-2-7-v1:0", "text", twelvelabs_embedding_response), + ("bedrock/twelvelabs.marengo-embed-2-7-v1:0", "image", twelvelabs_embedding_response), ], ) def test_bedrock_embedding_with_api_key_bearer_token(model, input_type, embed_response):