Add TwelveLabs marengo model (#14674)

litellm/constants.py

@@ -822,6 +822,7 @@ bedrock_embedding_models: set = set(
         "amazon.titan-embed-text-v1",
         "cohere.embed-english-v3",
         "cohere.embed-multilingual-v3",
+        "twelvelabs.marengo-embed-2-7-v1:0",
     ]
 )
 
@@ -1065,4 +1066,6 @@ SENTRY_PII_DENYLIST = [
 ]
 
 # CoroutineChecker cache configuration
-COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000))
+COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
+    os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)
+)

litellm/llms/bedrock/embed/embedding.py

@@ -4,8 +4,8 @@ Handles embedding calls to Bedrock's `/invoke` endpoint
 
 import copy
 import json
-from typing import Any, Callable, List, Optional, Tuple, Union
 import urllib.parse
+from typing import Any, Callable, List, Optional, Tuple, Union
 
 import httpx
 
@@ -18,7 +18,11 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
 )
 from litellm.secret_managers.main import get_secret
-from litellm.types.llms.bedrock import AmazonEmbeddingRequest, CohereEmbeddingRequest
+from litellm.types.llms.bedrock import (
+    AmazonEmbeddingRequest,
+    CohereEmbeddingRequest,
+    TwelveLabsMarengoEmbeddingRequest,
+)
 from litellm.types.utils import EmbeddingResponse
 
 from ..base_aws_llm import BaseAWSLLM
@@ -29,6 +33,7 @@ from .amazon_titan_multimodal_transformation import (
 )
 from .amazon_titan_v2_transformation import AmazonTitanV2Config
 from .cohere_transformation import BedrockCohereEmbeddingConfig
+from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
 
 
 class BedrockEmbedding(BaseAWSLLM):
@@ -164,16 +169,16 @@ class BedrockEmbedding(BaseAWSLLM):
             headers = {"Content-Type": "application/json"}
             if extra_headers is not None:
                 headers = {"Content-Type": "application/json", **extra_headers}
-                
+
             prepped = self.get_request_headers(
-                 credentials=credentials,
-                 aws_region_name=aws_region_name,
-                 extra_headers=extra_headers,
-                 endpoint_url=endpoint_url,
-                 data=json.dumps(data),
-                 headers=headers,
-                 api_key=api_key
-             )
+                credentials=credentials,
+                aws_region_name=aws_region_name,
+                extra_headers=extra_headers,
+                endpoint_url=endpoint_url,
+                data=json.dumps(data),
+                headers=headers,
+                api_key=api_key,
+            )
 
             ## LOGGING
             logging_obj.pre_call(
@@ -248,16 +253,16 @@ class BedrockEmbedding(BaseAWSLLM):
             headers = {"Content-Type": "application/json"}
             if extra_headers is not None:
                 headers = {"Content-Type": "application/json", **extra_headers}
-                
+
             prepped = self.get_request_headers(
-                 credentials=credentials,
-                 aws_region_name=aws_region_name,
-                 extra_headers=extra_headers,
-                 endpoint_url=endpoint_url,
-                 data=json.dumps(data),
-                 headers=headers,
-                 api_key=api_key,
-             )
+                credentials=credentials,
+                aws_region_name=aws_region_name,
+                extra_headers=extra_headers,
+                endpoint_url=endpoint_url,
+                data=json.dumps(data),
+                headers=headers,
+                api_key=api_key,
+            )
 
             ## LOGGING
             logging_obj.pre_call(
@@ -336,7 +341,7 @@ class BedrockEmbedding(BaseAWSLLM):
         ### TRANSFORMATION ###
         unencoded_model_id = (
             optional_params.pop("model_id", None) or model
-        ) # default to model if not passed
+        )  # default to model if not passed
         modelId = urllib.parse.quote(unencoded_model_id, safe="")
         aws_region_name = self._get_aws_region_name(
             optional_params=optional_params,
@@ -394,6 +399,17 @@ class BedrockEmbedding(BaseAWSLLM):
                         )
                     )
                 batch_data.append(transformed_request)
+        elif provider == "twelvelabs" and model in [
+            "twelvelabs.marengo-embed-2-7-v1:0",
+        ]:
+            batch_data = []
+            for i in input:
+                twelvelabs_request: (
+                    TwelveLabsMarengoEmbeddingRequest
+                ) = TwelveLabsMarengoEmbeddingConfig()._transform_request(
+                    input=i, inference_params=inference_params
+                )
+                batch_data.append(twelvelabs_request)
 
         ### SET RUNTIME ENDPOINT ###
         endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
@@ -445,7 +461,7 @@ class BedrockEmbedding(BaseAWSLLM):
         headers = {"Content-Type": "application/json"}
         if extra_headers is not None:
             headers = {"Content-Type": "application/json", **extra_headers}
-        
+
         prepped = self.get_request_headers(
             credentials=credentials,
             aws_region_name=aws_region_name,

litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py

@@ -0,0 +1,131 @@
+"""
+Transformation logic from OpenAI /v1/embeddings format to Bedrock TwelveLabs Marengo /invoke format.
+
+Why separate file? Make it easy to see how transformation works
+
+Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
+"""
+
+from typing import List
+
+from litellm.types.llms.bedrock import (
+    TwelveLabsMarengoEmbeddingRequest,
+)
+from litellm.types.utils import Embedding, EmbeddingResponse, Usage
+from litellm.utils import get_base64_str, is_base64_encoded
+
+
+class TwelveLabsMarengoEmbeddingConfig:
+    """
+    Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo.html
+
+    Supports text and image inputs for Phase 1.
+    Video and audio support will be added in Phase 2.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_supported_openai_params(self) -> List[str]:
+        return ["encoding_format", "textTruncate", "embeddingOption"]
+
+    def map_openai_params(
+        self, non_default_params: dict, optional_params: dict
+    ) -> dict:
+        for k, v in non_default_params.items():
+            if k == "encoding_format":
+                # TwelveLabs doesn't have encoding_format, but we can map it to embeddingOption
+                if v == "float":
+                    optional_params["embeddingOption"] = ["visual-text", "visual-image"]
+            elif k == "textTruncate":
+                optional_params["textTruncate"] = v
+            elif k == "embeddingOption":
+                optional_params["embeddingOption"] = v
+        return optional_params
+
+    def _transform_request(
+        self, input: str, inference_params: dict
+    ) -> TwelveLabsMarengoEmbeddingRequest:
+        """
+        Transform OpenAI-style input to TwelveLabs Marengo format.
+        Phase 1: Supports text and image inputs only.
+        """
+        # Check if input is base64 encoded image
+        is_encoded = is_base64_encoded(input)
+
+        if is_encoded:
+            # Image input
+            b64_str = get_base64_str(input)
+            transformed_request = TwelveLabsMarengoEmbeddingRequest(
+                inputType="image", mediaSource={"base64String": b64_str}
+            )
+        else:
+            # Text input
+            transformed_request = TwelveLabsMarengoEmbeddingRequest(
+                inputType="text", inputText=input
+            )
+
+            # Set default textTruncate if not specified
+            if "textTruncate" not in inference_params:
+                transformed_request["textTruncate"] = "end"
+
+        # Set default embedding options for Phase 1 (text and image)
+        if "embeddingOption" not in inference_params:
+            if is_encoded:
+                # For images, return both visual-text and visual-image embeddings
+                transformed_request["embeddingOption"] = ["visual-text", "visual-image"]
+            else:
+                # For text, return visual-text embedding
+                transformed_request["embeddingOption"] = ["visual-text"]
+
+        # Apply any additional inference parameters
+        for k, v in inference_params.items():
+            if k not in [
+                "inputType",
+                "inputText",
+                "mediaSource",
+            ]:  # Don't override core fields
+                transformed_request[k] = v  # type: ignore
+
+        return transformed_request
+
+    def _transform_response(
+        self, response_list: List[dict], model: str
+    ) -> EmbeddingResponse:
+        """
+        Transform TwelveLabs response to OpenAI format.
+        Handles multiple embedding types in the response.
+        """
+        embeddings: List[Embedding] = []
+        total_tokens = 0
+
+        for response in response_list:
+            if "embedding" in response:
+                # Single embedding response
+                embedding = Embedding(
+                    embedding=response["embedding"],
+                    index=len(embeddings),
+                    object="embedding",
+                )
+                embeddings.append(embedding)
+
+                # Estimate token count (rough approximation)
+                if "inputTextTokenCount" in response:
+                    total_tokens += response["inputTextTokenCount"]
+                else:
+                    # Rough estimate: 1 token per 4 characters for text
+                    total_tokens += len(response.get("inputText", "")) // 4
+            elif "embeddings" in response:
+                # Multiple embeddings response (from video/audio)
+                for i, emb in enumerate(response["embeddings"]):
+                    embedding = Embedding(
+                        embedding=emb["embedding"],
+                        index=len(embeddings),
+                        object="embedding",
+                    )
+                    embeddings.append(embedding)
+                    total_tokens += len(emb["embedding"]) // 4  # Rough estimate
+
+        usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
+
+        return EmbeddingResponse(data=embeddings, model=model, usage=usage)

litellm/types/llms/bedrock.py

@@ -364,6 +364,35 @@ class AmazonTitanMultimodalEmbeddingResponse(TypedDict):
     message: str  # Specifies any errors that occur during generation.
 
 
+# TwelveLabs Marengo Embed 2.7 types
+TWELVELABS_EMBEDDING_INPUT_TYPES = Literal["text", "image", "video", "audio"]
+TWELVELABS_EMBEDDING_OPTIONS = Literal["visual-text", "visual-image", "audio"]
+
+
+class TwelveLabsMediaSource(TypedDict, total=False):
+    base64String: str
+    s3Location: dict  # {"uri": str, "bucketOwner": str}
+
+
+class TwelveLabsMarengoEmbeddingRequest(TypedDict, total=False):
+    inputType: Required[TWELVELABS_EMBEDDING_INPUT_TYPES]
+    inputText: str
+    mediaSource: TwelveLabsMediaSource
+    textTruncate: Literal["end", "none"]
+    startSec: float
+    lengthSec: float
+    useFixedLengthSec: float
+    minClipSec: int
+    embeddingOption: List[TWELVELABS_EMBEDDING_OPTIONS]
+
+
+class TwelveLabsMarengoEmbeddingResponse(TypedDict):
+    embedding: List[float]
+    embeddingOption: TWELVELABS_EMBEDDING_OPTIONS
+    startSec: float
+    endSec: float
+
+
 AmazonEmbeddingRequest = Union[
     AmazonTitanMultimodalEmbeddingRequest,
     AmazonTitanV2EmbeddingRequest,

model_prices_and_context_window.json

@@ -296,6 +296,18 @@
         "output_cost_per_token": 0.0,
         "output_vector_size": 1024
     },
+    "twelvelabs.marengo-embed-2-7-v1:0": {
+        "input_cost_per_token": 7e-05,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 77,
+        "max_tokens": 77,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "output_vector_size": 1024,
+        "supports_embedding_image_input": true,
+        "supports_image_input": true,
+        "supports_multimodal_embedding": true
+    },
     "amazon.titan-text-express-v1": {
         "input_cost_per_token": 1.3e-06,
         "litellm_provider": "bedrock",

tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py

@@ -19,6 +19,13 @@ cohere_embedding_response = {
     "inputTextTokenCount": 10
 }
 
+twelvelabs_embedding_response = {
+    "embedding": [0.1, 0.2, 0.3],
+    "embeddingOption": "visual-text",
+    "startSec": 0.0,
+    "endSec": 1.0
+}
+
 # Test data
 test_input = "Hello world from litellm"
 test_image_base64 = "data:image/png,test_image_base64_data"
@@ -32,6 +39,8 @@ test_image_base64 = "data:image/png,test_image_base64_data"
         ("bedrock/amazon.titan-embed-image-v1", "image", titan_embedding_response),
         ("bedrock/cohere.embed-english-v3", "text", cohere_embedding_response),
         ("bedrock/cohere.embed-multilingual-v3", "text", cohere_embedding_response),
+        ("bedrock/twelvelabs.marengo-embed-2-7-v1:0", "text", twelvelabs_embedding_response),
+        ("bedrock/twelvelabs.marengo-embed-2-7-v1:0", "image", twelvelabs_embedding_response),
     ],
 )
 def test_bedrock_embedding_with_api_key_bearer_token(model, input_type, embed_response):