mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-10 22:24:51 +00:00
Merge pull request #9384 from BerriAI/litellm_prompt_management_custom
[Feat] - Allow building custom prompt management integration
This commit is contained in:
Ishaan Jaff
@@ -0,0 +1,194 @@
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Custom Prompt Management
|
||||
|
||||
Connect LiteLLM to your prompt management system with custom hooks.
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
<Image
|
||||
img={require('../../img/custom_prompt_management.png')}
|
||||
style={{width: '100%', display: 'block', margin: '2rem auto'}}
|
||||
/>
|
||||
|
||||
|
||||
## How it works
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Create Your Custom Prompt Manager
|
||||
|
||||
Create a class that inherits from `CustomPromptManagement` to handle prompt retrieval and formatting:
|
||||
|
||||
**Example Implementation**
|
||||
|
||||
Create a new file called `custom_prompt.py` and add this code. The key method here is `get_chat_completion_prompt` you can implement custom logic to retrieve and format prompts based on the `prompt_id` and `prompt_variables`.
|
||||
|
||||
```python
|
||||
from typing import List, Tuple, Optional
|
||||
from litellm.integrations.custom_prompt_management import CustomPromptManagement
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
class MyCustomPromptManagement(CustomPromptManagement):
|
||||
def get_chat_completion_prompt(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
non_default_params: dict,
|
||||
prompt_id: str,
|
||||
prompt_variables: Optional[dict],
|
||||
dynamic_callback_params: StandardCallbackDynamicParams,
|
||||
) -> Tuple[str, List[AllMessageValues], dict]:
|
||||
"""
|
||||
Retrieve and format prompts based on prompt_id.
|
||||
|
||||
Returns:
|
||||
- model: The model to use
|
||||
- messages: The formatted messages
|
||||
- non_default_params: Optional parameters like temperature
|
||||
"""
|
||||
# Example matching the diagram: Add system message for prompt_id "1234"
|
||||
if prompt_id == "1234":
|
||||
# Prepend system message while preserving existing messages
|
||||
new_messages = [
|
||||
{"role": "system", "content": "Be a good Bot!"},
|
||||
] + messages
|
||||
return model, new_messages, non_default_params
|
||||
|
||||
# Default: Return original messages if no prompt_id match
|
||||
return model, messages, non_default_params
|
||||
|
||||
prompt_management = MyCustomPromptManagement()
|
||||
```
|
||||
|
||||
### 2. Configure Your Prompt Manager in LiteLLM `config.yaml`
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: openai/gpt-4
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
litellm_settings:
|
||||
callbacks: custom_prompt.prompt_management # sets litellm.callbacks = [prompt_management]
|
||||
```
|
||||
|
||||
### 3. Start LiteLLM Gateway
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker Run">
|
||||
|
||||
Mount your `custom_logger.py` on the LiteLLM Docker container.
|
||||
|
||||
```shell
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
--name my-app \
|
||||
-v $(pwd)/my_config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/custom_logger.py:/app/custom_logger.py \
|
||||
my-app:latest \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="py" label="litellm pip">
|
||||
|
||||
```shell
|
||||
litellm --config config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### 4. Test Your Custom Prompt Manager
|
||||
|
||||
When you pass `prompt_id="1234"`, the custom prompt manager will add a system message "Be a good Bot!" to your conversation:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai" label="OpenAI Python v1.0.0+">
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(
|
||||
api_key="sk-1234",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="gemini-1.5-pro",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
prompt_id="1234"
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="langchain" label="Langchain">
|
||||
|
||||
```python
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.schema import HumanMessage
|
||||
|
||||
chat = ChatOpenAI(
|
||||
model="gpt-4",
|
||||
openai_api_key="sk-1234",
|
||||
openai_api_base="http://0.0.0.0:4000",
|
||||
extra_body={
|
||||
"prompt_id": "1234"
|
||||
}
|
||||
)
|
||||
|
||||
messages = []
|
||||
response = chat(messages)
|
||||
|
||||
print(response.content)
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="Curl">
|
||||
|
||||
```shell
|
||||
curl -X POST http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "gemini-1.5-pro",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"prompt_id": "1234"
|
||||
}'
|
||||
```
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
The request will be transformed from:
|
||||
```json
|
||||
{
|
||||
"model": "gemini-1.5-pro",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"prompt_id": "1234"
|
||||
}
|
||||
```
|
||||
|
||||
To:
|
||||
```json
|
||||
{
|
||||
"model": "gemini-1.5-pro",
|
||||
"messages": [
|
||||
{"role": "system", "content": "Be a good Bot!"},
|
||||
{"role": "user", "content": "hi"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# [BETA] Prompt Management
|
||||
# Prompt Management
|
||||
|
||||
:::info
|
||||
|
||||
@@ -12,9 +12,10 @@ This feature is currently in beta, and might change unexpectedly. We expect this
|
||||
|
||||
Run experiments or change the specific model (e.g. from gpt-4o to gpt4o-mini finetune) from your prompt management tool (e.g. Langfuse) instead of making changes in the application.
|
||||
|
||||
Supported Integrations:
|
||||
- [Langfuse](https://langfuse.com/docs/prompts/get-started)
|
||||
- [Humanloop](../observability/humanloop)
|
||||
| Supported Integrations | Link |
|
||||
|------------------------|------|
|
||||
| Langfuse | [Get Started](https://langfuse.com/docs/prompts/get-started) |
|
||||
| Humanloop | [Get Started](../observability/humanloop) |
|
||||
|
||||
## Quick Start
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 346 KiB |
@@ -365,8 +365,12 @@ const sidebars = {
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "doc",
|
||||
id: "proxy/prompt_management"
|
||||
type: "category",
|
||||
label: "[Beta] Prompt Management",
|
||||
items: [
|
||||
"proxy/prompt_management",
|
||||
"proxy/custom_prompt_management"
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
|
||||
Reference in New Issue
Block a user