diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md
index 34b213f0e2..b6d8c83652 100644
--- a/docs/my-website/docs/observability/langfuse_integration.md
+++ b/docs/my-website/docs/observability/langfuse_integration.md
@@ -11,6 +11,13 @@ Example trace in Langfuse using multiple models via LiteLLM:
+:::info
+
+For Langfuse v3, we recommend using the [Langfuse OTEL](./langfuse_otel_integration) integration.
+
+:::
+
+
## Usage with LiteLLM Proxy (LLM Gateway)
👉 [**Follow this link to start sending logs to langfuse with LiteLLM Proxy server**](../proxy/logging)
diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md
index 267738c300..c45c33f0f2 100644
--- a/docs/my-website/docs/observability/langfuse_otel_integration.md
+++ b/docs/my-website/docs/observability/langfuse_otel_integration.md
@@ -1,7 +1,14 @@
-# Langfuse OpenTelemetry Integration
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+import Image from '@theme/IdealImage';
+
+# 🪢 Langfuse OpenTelemetry Integration
The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and observability data to Langfuse using the OpenTelemetry protocol. This provides a standardized way to collect and analyze your LLM usage data.
+
+
## Features
- Automatic trace collection for all LiteLLM requests
@@ -108,15 +115,26 @@ litellm.callbacks = ["langfuse_otel"]
Add the integration to your proxy configuration:
+1. Add the credentials to your environment variables
+
+```bash
+export LANGFUSE_PUBLIC_KEY="pk-lf-..."
+export LANGFUSE_SECRET_KEY="sk-lf-..."
+export LANGFUSE_HOST="https://us.cloud.langfuse.com" # Default US region
+```
+
+2. Setup config.yaml
+
```yaml
# config.yaml
litellm_settings:
callbacks: ["langfuse_otel"]
+```
-environment_variables:
- LANGFUSE_PUBLIC_KEY: "pk-lf-..."
- LANGFUSE_SECRET_KEY: "sk-lf-..."
- LANGFUSE_HOST: "https://us.cloud.langfuse.com" # Default US region
+3. Run the proxy
+
+```bash
+litellm --config /path/to/config.yaml
```
## Data Collected
@@ -163,11 +181,24 @@ This is automatically handled by the integration - you just need to provide the
Enable verbose logging to see detailed information:
+
+
+
```python
import litellm
-litellm.set_verbose = True
+litellm._turn_on_debug()
```
+
+
+
+```bash
+export LITELLM_LOG="DEBUG"
+```
+
+
+
+
This will show:
- Endpoint resolution logic
- Authentication header creation
diff --git a/docs/my-website/docs/proxy/prompt_management.md b/docs/my-website/docs/proxy/prompt_management.md
index 8ea17425c8..fc35fc5ef3 100644
--- a/docs/my-website/docs/proxy/prompt_management.md
+++ b/docs/my-website/docs/proxy/prompt_management.md
@@ -210,6 +210,7 @@ These are the params you can pass to the `litellm.completion` function in SDK an
```
prompt_id: str # required
prompt_variables: Optional[dict] # optional
+prompt_version: Optional[int] # optional
langfuse_public_key: Optional[str] # optional
langfuse_secret: Optional[str] # optional
langfuse_secret_key: Optional[str] # optional
diff --git a/docs/my-website/img/langfuse_otel.png b/docs/my-website/img/langfuse_otel.png
new file mode 100644
index 0000000000..a91e337f2c
Binary files /dev/null and b/docs/my-website/img/langfuse_otel.png differ
diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py
index 5c75e452ab..29d9920da4 100644
--- a/litellm/integrations/anthropic_cache_control_hook.py
+++ b/litellm/integrations/anthropic_cache_control_hook.py
@@ -29,6 +29,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Apply cache control directives based on specified injection points.
@@ -80,10 +81,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# Case 1: Target by specific index
if targetted_index is not None:
if 0 <= targetted_index < len(messages):
- messages[
- targetted_index
- ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
- messages[targetted_index], control
+ messages[targetted_index] = (
+ AnthropicCacheControlHook._safe_insert_cache_control_in_message(
+ messages[targetted_index], control
+ )
)
# Case 2: Target by role
elif targetted_role is not None:
diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py
index 890482761a..a9cbc65e6f 100644
--- a/litellm/integrations/custom_logger.py
+++ b/litellm/integrations/custom_logger.py
@@ -89,6 +89,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -107,6 +108,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -408,9 +410,10 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
if len(text) > max_length
else text
)
-
- def _select_metadata_field(self, request_kwargs: Optional[Dict] = None) -> Optional[str]:
+ def _select_metadata_field(
+ self, request_kwargs: Optional[Dict] = None
+ ) -> Optional[str]:
"""
Select the metadata field to use for logging
@@ -418,9 +421,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
2. Otherwise, use `metadata`
"""
from litellm.constants import LITELLM_METADATA_FIELD, OLD_LITELLM_METADATA_FIELD
+
if request_kwargs is None:
return None
if LITELLM_METADATA_FIELD in request_kwargs:
return LITELLM_METADATA_FIELD
return OLD_LITELLM_METADATA_FIELD
-
\ No newline at end of file
diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py
index 061aadc3c0..86cd1dc9f7 100644
--- a/litellm/integrations/custom_prompt_management.py
+++ b/litellm/integrations/custom_prompt_management.py
@@ -19,6 +19,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
@@ -45,6 +46,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> PromptManagementClient:
raise NotImplementedError(
"Custom prompt management does not support compile prompt helper"
diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py
index c62ab1110f..9f43d80626 100644
--- a/litellm/integrations/humanloop.py
+++ b/litellm/integrations/humanloop.py
@@ -156,7 +156,12 @@ class HumanloopLogger(CustomLogger):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
- ) -> Tuple[str, List[AllMessageValues], dict,]:
+ prompt_version: Optional[int] = None,
+ ) -> Tuple[
+ str,
+ List[AllMessageValues],
+ dict,
+ ]:
humanloop_api_key = dynamic_callback_params.get(
"humanloop_api_key"
) or get_secret_str("HUMANLOOP_API_KEY")
diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py
index 6d7f927c3e..7fc222ff6c 100644
--- a/litellm/integrations/langfuse/langfuse_otel.py
+++ b/litellm/integrations/langfuse/langfuse_otel.py
@@ -1,6 +1,7 @@
import base64
import os
from typing import TYPE_CHECKING, Any, Union
+from urllib.parse import quote
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
@@ -9,10 +10,11 @@ from litellm.types.integrations.langfuse_otel import LangfuseOtelConfig
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
+ from litellm.integrations.opentelemetry import (
+ OpenTelemetryConfig as _OpenTelemetryConfig,
+ )
from litellm.types.integrations.arize import Protocol as _Protocol
- from litellm.integrations.opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig
-
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
@@ -54,7 +56,7 @@ class LangfuseOtelLogger:
"""
public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", None)
secret_key = os.environ.get("LANGFUSE_SECRET_KEY", None)
-
+
if not public_key or not secret_key:
raise ValueError(
"LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set for Langfuse OpenTelemetry integration."
@@ -62,7 +64,7 @@ class LangfuseOtelLogger:
# Determine endpoint - default to US cloud
langfuse_host = os.environ.get("LANGFUSE_HOST", None)
-
+
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it
if not langfuse_host.startswith("http"):
@@ -77,13 +79,13 @@ class LangfuseOtelLogger:
# Create Basic Auth header
auth_string = f"{public_key}:{secret_key}"
auth_header = base64.b64encode(auth_string.encode()).decode()
- otlp_auth_headers = f"Authorization=Basic {auth_header}"
+ # URL encode the entire header value as required by OpenTelemetry specification
+ otlp_auth_headers = f"Authorization={quote(f'Basic {auth_header}')}"
# Set standard OTEL environment variables
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
return LangfuseOtelConfig(
- otlp_auth_headers=otlp_auth_headers,
- protocol="otlp_http"
- )
\ No newline at end of file
+ otlp_auth_headers=otlp_auth_headers, protocol="otlp_http"
+ )
diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py
index 8fe9cb63de..58698ef35a 100644
--- a/litellm/integrations/langfuse/langfuse_prompt_management.py
+++ b/litellm/integrations/langfuse/langfuse_prompt_management.py
@@ -134,8 +134,14 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_prompt_id: str,
langfuse_client: LangfuseClass,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> PROMPT_CLIENT:
- return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label)
+
+ prompt_client = langfuse_client.get_prompt(
+ langfuse_prompt_id, label=prompt_label, version=prompt_version
+ )
+
+ return prompt_client
def _compile_prompt(
self,
@@ -180,7 +186,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
- ) -> Tuple[str, List[AllMessageValues], dict,]:
+ prompt_version: Optional[int] = None,
+ ) -> Tuple[
+ str,
+ List[AllMessageValues],
+ dict,
+ ]:
return self.get_chat_completion_prompt(
model,
messages,
@@ -189,6 +200,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_variables,
dynamic_callback_params,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
def should_run_prompt_management(
@@ -203,7 +215,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_host=dynamic_callback_params.get("langfuse_host"),
)
langfuse_prompt_client = self._get_prompt_from_id(
- langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client
+ langfuse_prompt_id=prompt_id,
+ langfuse_client=langfuse_client,
)
return langfuse_prompt_client is not None
@@ -213,6 +226,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> PromptManagementClient:
langfuse_client = langfuse_client_init(
langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"),
@@ -224,6 +238,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_prompt_id=prompt_id,
langfuse_client=langfuse_client,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
## SET PROMPT
diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py
index c9e7adbccb..4a8bcd2e24 100644
--- a/litellm/integrations/prompt_management_base.py
+++ b/litellm/integrations/prompt_management_base.py
@@ -34,6 +34,7 @@ class PromptManagementBase(ABC):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> PromptManagementClient:
pass
@@ -51,12 +52,14 @@ class PromptManagementBase(ABC):
client_messages: List[AllMessageValues],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> PromptManagementClient:
compiled_prompt_client = self._compile_prompt_helper(
prompt_id=prompt_id,
prompt_variables=prompt_variables,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
try:
@@ -86,6 +89,7 @@ class PromptManagementBase(ABC):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
if prompt_id is None:
raise ValueError("prompt_id is required for Prompt Management Base class")
@@ -100,6 +104,7 @@ class PromptManagementBase(ABC):
client_messages=messages,
dynamic_callback_params=dynamic_callback_params,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
completed_messages = prompt_template["completed_messages"] or messages
diff --git a/litellm/integrations/vector_store_integrations/bedrock_vector_store.py b/litellm/integrations/vector_store_integrations/bedrock_vector_store.py
index a00acefb6a..d3ba3a8ebd 100644
--- a/litellm/integrations/vector_store_integrations/bedrock_vector_store.py
+++ b/litellm/integrations/vector_store_integrations/bedrock_vector_store.py
@@ -77,6 +77,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Retrieves the context from the Bedrock Knowledge Base and appends it to the messages.
@@ -129,9 +130,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
)
)
- litellm_logging_obj.model_call_details[
- "vector_store_request_metadata"
- ] = vector_store_request_metadata
+ litellm_logging_obj.model_call_details["vector_store_request_metadata"] = (
+ vector_store_request_metadata
+ )
return model, messages, non_default_params
@@ -143,9 +144,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
"""
Transform a BedrockKBResponse to a VectorStoreSearchResponse
"""
- retrieval_results: Optional[
- List[BedrockKBRetrievalResult]
- ] = bedrock_kb_response.get("retrievalResults", None)
+ retrieval_results: Optional[List[BedrockKBRetrievalResult]] = (
+ bedrock_kb_response.get("retrievalResults", None)
+ )
vector_store_search_response: VectorStoreSearchResponse = (
VectorStoreSearchResponse(search_query=query, data=[])
)
diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py
index a00acefb6a..d3ba3a8ebd 100644
--- a/litellm/integrations/vector_stores/bedrock_vector_store.py
+++ b/litellm/integrations/vector_stores/bedrock_vector_store.py
@@ -77,6 +77,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
litellm_logging_obj: LiteLLMLoggingObj,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Retrieves the context from the Bedrock Knowledge Base and appends it to the messages.
@@ -129,9 +130,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
)
)
- litellm_logging_obj.model_call_details[
- "vector_store_request_metadata"
- ] = vector_store_request_metadata
+ litellm_logging_obj.model_call_details["vector_store_request_metadata"] = (
+ vector_store_request_metadata
+ )
return model, messages, non_default_params
@@ -143,9 +144,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM):
"""
Transform a BedrockKBResponse to a VectorStoreSearchResponse
"""
- retrieval_results: Optional[
- List[BedrockKBRetrievalResult]
- ] = bedrock_kb_response.get("retrievalResults", None)
+ retrieval_results: Optional[List[BedrockKBRetrievalResult]] = (
+ bedrock_kb_response.get("retrievalResults", None)
+ )
vector_store_search_response: VectorStoreSearchResponse = (
VectorStoreSearchResponse(search_query=query, data=[])
)
diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py
index e5a19e7bdd..c425319b4d 100644
--- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py
+++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py
@@ -18,6 +18,7 @@ def initialize_standard_callback_dynamic_params(
_supported_callback_params = (
StandardCallbackDynamicParams.__annotations__.keys()
)
+
for param in _supported_callback_params:
if param in kwargs:
_param_value = kwargs.pop(param)
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index f7be8b1bb5..9ff7903540 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -435,6 +435,7 @@ class Logging(LiteLLMLoggingBaseClass):
checks if langfuse_secret_key, gcs_bucket_name in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
"""
+
return _initialize_standard_callback_dynamic_params(kwargs)
def initialize_standard_built_in_tools_params(
@@ -556,6 +557,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_variables: Optional[dict],
prompt_management_logger: Optional[CustomLogger] = None,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@@ -577,6 +579,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_variables=prompt_variables,
dynamic_callback_params=self.standard_callback_dynamic_params,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
self.messages = messages
return model, messages, non_default_params
@@ -591,6 +594,7 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_management_logger: Optional[CustomLogger] = None,
tools: Optional[List[Dict]] = None,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
custom_logger = (
prompt_management_logger
@@ -614,6 +618,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_logging_obj=self,
tools=tools,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
self.messages = messages
return model, messages, non_default_params
@@ -2953,8 +2958,6 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
import sentry_sdk
from sentry_sdk.scrubber import EventScrubber
-
-
sentry_sdk_instance = sentry_sdk
sentry_trace_rate = (
os.environ.get("SENTRY_API_TRACE_RATE")
@@ -2974,8 +2977,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
),
send_default_pii=False, # Prevent sending Personal Identifiable Information
event_scrubber=EventScrubber(
- denylist=SENTRY_DENYLIST,
- pii_denylist=SENTRY_PII_DENYLIST
+ denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST
),
)
capture_exception = sentry_sdk_instance.capture_exception
@@ -3033,6 +3035,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
s3Logger = S3Logger()
elif callback == "wandb":
from litellm.integrations.weights_biases import WeightsBiasesLogger
+
weightsBiasesLogger = WeightsBiasesLogger()
elif callback == "logfire":
logfireLogger = LogfireLogger()
@@ -3087,6 +3090,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
return _openmeter_logger # type: ignore
elif logging_integration == "braintrust":
from litellm.integrations.braintrust_logging import BraintrustLogger
+
for callback in _in_memory_loggers:
if isinstance(callback, BraintrustLogger):
return callback # type: ignore
@@ -3453,6 +3457,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
return callback
elif logging_integration == "braintrust":
from litellm.integrations.braintrust_logging import BraintrustLogger
+
for callback in _in_memory_loggers:
if isinstance(callback, BraintrustLogger):
return callback
diff --git a/litellm/main.py b/litellm/main.py
index da6db8e835..5d0662060d 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -438,6 +438,7 @@ async def acompletion(
prompt_variables=kwargs.get("prompt_variables", None),
tools=tools,
prompt_label=kwargs.get("prompt_label", None),
+ prompt_version=kwargs.get("prompt_version", None),
)
#########################################################
# if the chat completion logging hook removed all tools,
@@ -1062,6 +1063,7 @@ def completion( # type: ignore # noqa: PLR0915
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=kwargs.get("prompt_label", None),
+ prompt_version=kwargs.get("prompt_version", None),
)
try:
@@ -1950,7 +1952,7 @@ def completion( # type: ignore # noqa: PLR0915
or get_secret("MISTRAL_API_BASE")
or "https://api.mistral.ai/v1"
)
-
+
response = base_llm_http_handler.completion(
model=model,
messages=messages,
@@ -2615,9 +2617,7 @@ def completion( # type: ignore # noqa: PLR0915
api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE")
new_params = deepcopy(optional_params)
- if (
- vertex_partner_models_chat_completion.is_vertex_partner_model(model)
- ):
+ if vertex_partner_models_chat_completion.is_vertex_partner_model(model):
model_response = vertex_partner_models_chat_completion.completion(
model=model,
messages=messages,
diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html
deleted file mode 100644
index be3a989197..0000000000
--- a/litellm/proxy/_experimental/out/onboarding.html
+++ /dev/null
@@ -1 +0,0 @@
-LiteLLM Dashboard
\ No newline at end of file
diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml
index 8be231a1fe..afaac02c08 100644
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@@ -3,11 +3,11 @@ model_list:
litellm_params:
model: openai/gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
-
-guardrails:
- - guardrail_name: azure-prompt-shield
+ - model_name: langfuse-model
litellm_params:
- guardrail: azure/prompt_shield
- mode: pre_call # "During_call" is also available
- api_key: os.environ/AZURE_GUARDRAIL_API_KEY
- api_base: os.environ/AZURE_GUARDRAIL_API_BASE
\ No newline at end of file
+ model: langfuse/langfuse-model
+ prompt_id: test-chat-prompt
+ prompt_version: 4
+
+litellm_settings:
+ callbacks: ["langfuse_otel"]
\ No newline at end of file
diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py
index 8cf20da5e9..cae5890b6c 100644
--- a/litellm/proxy/custom_prompt_management.py
+++ b/litellm/proxy/custom_prompt_management.py
@@ -16,6 +16,7 @@ class X42PromptManagement(CustomPromptManagement):
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
"""
Returns:
diff --git a/litellm/router.py b/litellm/router.py
index dc12b924ff..4ed8658058 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1735,6 +1735,7 @@ class Router:
kwargs: Dict[str, Any],
):
litellm_logging_object = kwargs.get("litellm_logging_obj", None)
+
if litellm_logging_object is None:
litellm_logging_object, kwargs = function_setup(
**{
@@ -1770,6 +1771,10 @@ class Router:
"litellm_params"
].get("prompt_label", None)
+ prompt_version = kwargs.get(
+ "prompt_version", None
+ ) or prompt_management_deployment["litellm_params"].get("prompt_version", None)
+
if prompt_id is None or not isinstance(prompt_id, str):
raise ValueError(
f"Prompt ID is not set or not a string. Got={prompt_id}, type={type(prompt_id)}"
@@ -1790,6 +1795,7 @@ class Router:
prompt_id=prompt_id,
prompt_variables=prompt_variables,
prompt_label=prompt_label,
+ prompt_version=prompt_version,
)
kwargs = {**data, **kwargs, **optional_params}
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index b9d24d9455..3f5ea0785c 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2072,6 +2072,9 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
langfuse_secret_key: Optional[str]
langfuse_host: Optional[str]
+ # Langfuse prompt version
+ langfuse_prompt_version: Optional[int]
+
# GCS dynamic params
gcs_bucket_name: Optional[str]
gcs_path_service_account: Optional[str]
@@ -2112,6 +2115,7 @@ all_litellm_params = [
"prompt_id",
"provider_specific_header",
"prompt_variables",
+ "prompt_version",
"api_base",
"force_timeout",
"logger_fn",
diff --git a/tests/test_litellm/integrations/langfuse/test_langfuse_prompt_management.py b/tests/test_litellm/integrations/langfuse/test_langfuse_prompt_management.py
new file mode 100644
index 0000000000..7a5b8bfb1c
--- /dev/null
+++ b/tests/test_litellm/integrations/langfuse/test_langfuse_prompt_management.py
@@ -0,0 +1,32 @@
+import os
+from unittest.mock import patch
+
+from litellm.integrations.langfuse.langfuse_prompt_management import (
+ LangfusePromptManagement,
+)
+
+
+class TestLangfusePromptManagement:
+ def test_get_prompt_from_id(self):
+ langfuse_prompt_management = LangfusePromptManagement()
+ with patch.object(
+ langfuse_prompt_management, "should_run_prompt_management"
+ ) as mock_should_run_prompt_management, patch.object(
+ langfuse_prompt_management, "_get_prompt_from_id"
+ ) as mock_get_prompt_from_id:
+ mock_should_run_prompt_management.return_value = True
+ chat_completion_prompt = (
+ langfuse_prompt_management.get_chat_completion_prompt(
+ model="langfuse/langfuse-model",
+ messages=[{"role": "user", "content": "Hello, how are you?"}],
+ non_default_params={},
+ prompt_id="test-chat-prompt",
+ prompt_variables={},
+ dynamic_callback_params={},
+ prompt_version=4,
+ )
+ )
+
+ mock_get_prompt_from_id.assert_called_once()
+ print(mock_get_prompt_from_id.call_args.kwargs)
+ assert mock_get_prompt_from_id.call_args.kwargs["prompt_version"] == 4
diff --git a/tests/test_litellm/integrations/test_custom_prompt_management.py b/tests/test_litellm/integrations/test_custom_prompt_management.py
index f5855abf71..f01462070f 100644
--- a/tests/test_litellm/integrations/test_custom_prompt_management.py
+++ b/tests/test_litellm/integrations/test_custom_prompt_management.py
@@ -33,7 +33,8 @@ class TestCustomPromptManagement(CustomPromptManagement):
prompt_id: Optional[str],
prompt_variables: Optional[dict],
dynamic_callback_params: StandardCallbackDynamicParams,
- prompt_label: Optional[str],
+ prompt_label: Optional[str] = None,
+ prompt_version: Optional[int] = None,
) -> Tuple[str, List[AllMessageValues], dict]:
print(
"TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ",