diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md
index 9731db6e75..95c922cce8 100644
--- a/docs/my-website/docs/integrations/index.md
+++ b/docs/my-website/docs/integrations/index.md
@@ -2,4 +2,17 @@
This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
Click into each section to learn more about the integrations.
\ No newline at end of file
diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md
new file mode 100644
index 0000000000..2afb82542f
--- /dev/null
+++ b/docs/my-website/docs/integrations/letta.md
@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+
+
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+
+ - model_name: claude-3-sonnet
+ litellm_params:
+ model: anthropic/claude-3-sonnet-20240229
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: azure/gpt-35-turbo
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+ model="gpt-4", # This should match a model from your LiteLLM config
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000", # Your LiteLLM proxy URL
+ context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+ embedding_endpoint_type="openai",
+ embedding_endpoint="http://localhost:4000",
+ embedding_model="text-embedding-ada-002"
+)
+```
+
+
+
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+ def __init__(self, model: str):
+ self.model = model
+
+ def chat_completions_create(self, messages: List[Dict], **kwargs):
+ # Use LiteLLM SDK for completion
+ response = litellm.completion(
+ model=self.model,
+ messages=messages,
+ **kwargs
+ )
+ return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+ model="gpt-4", # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+
+
+
+### 3. Create and Use a Letta Agent
+
+
+
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+
+
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint_type="openai", # Using OpenAI-compatible endpoint
+ model_endpoint="http://localhost:4000",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+ model="openai/gpt-4", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+### Function Calling with Tools
+
+
+
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4", # Direct model specification
+ model_endpoint_type="openai",
+ context_window=8192
+ ),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+## Authentication
+
+
+
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ model_wrapper="openai",
+ context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+ master_key: "your-master-key"
+
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192,
+ api_key="your-master-key" # Proxy master key
+)
+```
+
+
+
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key"
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY") # Default key
+litellm.set_verbose = True # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+ model="openai/gpt-4", # Will use OPENAI_API_KEY automatically
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+ model="azure/gpt-35-turbo",
+ model_endpoint_type="openai",
+ context_window=4096
+)
+```
+
+
+
+
+## Load Balancing and Fallbacks
+
+
+
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ tpm: 40000
+ rpm: 500
+
+ - model_name: gpt-4 # Same model name for fallback
+ litellm_params:
+ model: azure/gpt-4
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+ tpm: 80000
+ rpm: 800
+
+router_settings:
+ routing_strategy: "usage-based-routing"
+ fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+
+
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+ model_list=[
+ {
+ "model_name": "gpt-4",
+ "litellm_params": {
+ "model": "openai/gpt-4",
+ "api_key": os.environ["OPENAI_API_KEY"]
+ },
+ "tpm": 40000,
+ "rpm": 500
+ },
+ {
+ "model_name": "gpt-4", # Same name for fallback
+ "litellm_params": {
+ "model": "azure/gpt-4",
+ "api_key": os.environ["AZURE_API_KEY"],
+ "api_base": os.environ["AZURE_API_BASE"],
+ "api_version": "2023-07-01-preview"
+ },
+ "tpm": 80000,
+ "rpm": 800
+ }
+ ],
+ fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+ routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+ return router.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+
+
+
+## Monitoring and Observability
+
+
+
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+ # ... your models
+
+litellm_settings:
+ success_callback: ["langfuse"] # or other observability tools
+
+environment_variables:
+ LANGFUSE_PUBLIC_KEY: "your-key"
+ LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+
+
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+ # Custom logging logic here if needed
+ return response
+ except Exception as e:
+ # Custom error handling
+ print(f"LLM call failed: {e}")
+ raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+
+
+
+## Example: Multi-Agent System
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="gpt-4",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4",
+ model_endpoint_type="openai"
+ )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+ name="reviewer",
+ system="You are an editor. Review and improve content quality.",
+ llm_config=LLMConfig(
+ model="openai/gpt-3.5-turbo",
+ model_endpoint_type="openai"
+ )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ draft_article = write_response.messages[-1].text
+
+ # Review phase
+ review_response = client.user_message(
+ agent_id=agents['reviewer'].id,
+ message=f"Please review and improve this article:\n\n{draft_article}"
+ )
+
+ return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+## Best Practices
+
+
+
+
+1. **Model Selection**: Use appropriate models for different tasks:
+ - Claude for analysis and reasoning
+ - GPT-4 for creative tasks
+ - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+ - Set appropriate rate limits and timeouts
+ - Use fallbacks for reliability
+ - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**:
+ - Use the proxy's budgeting features to control costs
+ - Set up rate limiting per user/team
+ - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+
+
+
+1. **Model Selection**: Choose models based on task requirements:
+ - Use `openai/gpt-4` for complex reasoning
+ - Use `anthropic/claude-3-sonnet-20240229` for analysis
+ - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+ ```python
+ import litellm
+ from litellm import completion
+
+ # Set up retry logic
+ litellm.num_retries = 3
+ litellm.request_timeout = 60
+
+ # Custom error handling
+ def safe_completion(**kwargs):
+ try:
+ return completion(**kwargs)
+ except Exception as e:
+ print(f"LLM call failed: {e}")
+ # Implement fallback logic
+ return completion(model="openai/gpt-3.5-turbo", **kwargs)
+ ```
+
+3. **Cost Management**:
+ - Use cheaper models for non-critical tasks
+ - Implement token counting and budgets
+ - Cache responses when appropriate
+
+4. **Performance**:
+ - Use async operations for concurrent requests
+ - Implement connection pooling
+ - Monitor response times
+
+5. **Security**:
+ - Store API keys securely (environment variables)
+ - Rotate keys regularly
+ - Implement rate limiting
+
+
+
+
+## Troubleshooting
+
+
+
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}]
+ }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+
+
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+ response = litellm.completion(
+ model="openai/gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+ print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+ print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=[{"role": "user", "content": "Test"}],
+ max_tokens=10
+ )
+ print(f"✓ {model} working")
+ except Exception as e:
+ print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+
+
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)
\ No newline at end of file
diff --git a/docs/my-website/docs/proxy/custom_auth.md b/docs/my-website/docs/proxy/custom_auth.md
index 3787f9bdd7..812b80d3e9 100644
--- a/docs/my-website/docs/proxy/custom_auth.md
+++ b/docs/my-website/docs/proxy/custom_auth.md
@@ -21,6 +21,169 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
raise Exception
```
+## UserAPIKeyAuth Fields Reference
+
+The `UserAPIKeyAuth` object supports the following fields for comprehensive auth configuration:
+
+### Core Authentication Fields
+```python
+UserAPIKeyAuth(
+ # Basic auth fields
+ api_key: Optional[str] = None, # The API key (will be hashed automatically)
+ token: Optional[str] = None, # Hashed token for internal use
+ key_name: Optional[str] = None, # Human-readable key name
+ key_alias: Optional[str] = None, # Key alias for identification
+
+ # User identification
+ user_id: Optional[str] = None, # Unique user identifier
+ user_email: Optional[str] = None, # User email address
+ user_role: Optional[LitellmUserRoles] = None, # User role (PROXY_ADMIN, INTERNAL_USER, etc.)
+
+ # Team/Organization
+ team_id: Optional[str] = None, # Team identifier
+ team_alias: Optional[str] = None, # Team display name
+ org_id: Optional[str] = None, # Organization identifier
+)
+```
+
+### Budget and Spend Tracking
+```python
+UserAPIKeyAuth(
+ # User budgets
+ max_budget: Optional[float] = None, # Maximum budget for the key
+ spend: float = 0.0, # Current spend amount
+ soft_budget: Optional[float] = None, # Soft budget limit (warnings)
+ model_max_budget: Dict = {}, # Per-model budget limits
+ model_spend: Dict = {}, # Per-model spend tracking
+
+ # Team budgets
+ team_max_budget: Optional[float] = None, # Team's maximum budget
+ team_spend: Optional[float] = None, # Team's current spend
+ team_member_spend: Optional[float] = None, # This user's spend within the team
+
+ # Budget timing
+ budget_duration: Optional[str] = None, # Budget reset period
+ budget_reset_at: Optional[datetime] = None, # When budget resets
+)
+```
+
+### Rate Limiting
+```python
+UserAPIKeyAuth(
+ # User limits
+ tpm_limit: Optional[int] = None, # Tokens per minute limit
+ rpm_limit: Optional[int] = None, # Requests per minute limit
+ user_tpm_limit: Optional[int] = None, # User-specific TPM limit
+ user_rpm_limit: Optional[int] = None, # User-specific RPM limit
+
+ # Team limits
+ team_tpm_limit: Optional[int] = None, # Team TPM limit
+ team_rpm_limit: Optional[int] = None, # Team RPM limit
+ team_member_tpm_limit: Optional[int] = None, # Per-member TPM limit
+ team_member_rpm_limit: Optional[int] = None, # Per-member RPM limit
+
+ # Per-model limits
+ rpm_limit_per_model: Optional[Dict[str, int]] = None, # RPM limits by model
+ tpm_limit_per_model: Optional[Dict[str, int]] = None, # TPM limits by model
+)
+```
+
+### End User Tracking
+```python
+UserAPIKeyAuth(
+ # End user identification and limits
+ end_user_id: Optional[str] = None, # End user identifier
+ end_user_tpm_limit: Optional[int] = None, # End user TPM limit
+ end_user_rpm_limit: Optional[int] = None, # End user RPM limit
+ end_user_max_budget: Optional[float] = None, # End user budget limit
+)
+```
+
+### Model and Route Access
+```python
+UserAPIKeyAuth(
+ # Model access control
+ models: List = [], # Allowed models list
+ team_models: List = [], # Team's allowed models
+ aliases: Dict = {}, # Model aliases
+
+ # Route permissions
+ allowed_routes: Optional[list] = [], # Allowed API routes
+ allowed_cache_controls: Optional[list] = [], # Cache control permissions
+ permissions: Dict = {}, # General permissions
+)
+```
+
+### Advanced Configuration
+```python
+UserAPIKeyAuth(
+ # Request handling
+ max_parallel_requests: Optional[int] = None, # Concurrent request limit
+ allowed_model_region: Optional[AllowedModelRegion] = None, # Geographic restrictions
+
+ # Expiration and status
+ expires: Optional[Union[str, datetime]] = None, # Key expiration
+ blocked: Optional[bool] = None, # Whether key is blocked
+
+ # Metadata and configuration
+ metadata: Dict = {}, # Custom metadata
+ config: Dict = {}, # Configuration settings
+ team_metadata: Optional[Dict] = None, # Team metadata
+
+ # Internal tracking
+ request_route: Optional[str] = None, # Current request route
+ last_refreshed_at: Optional[float] = None, # Cache refresh timestamp
+)
+```
+
+### Complete Example
+
+```python
+from datetime import datetime, timedelta
+from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
+
+async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
+ try:
+ # Example: Comprehensive auth configuration
+ if api_key.startswith("sk-admin-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="admin_user_123",
+ user_email="admin@company.com",
+ user_role=LitellmUserRoles.PROXY_ADMIN,
+ team_id="admin_team",
+ team_alias="Administrative Team",
+ max_budget=1000.0,
+ soft_budget=800.0,
+ tpm_limit=10000,
+ rpm_limit=100,
+ models=["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"],
+ allowed_routes=["/chat/completions", "/embeddings"],
+ expires=datetime.now() + timedelta(days=30),
+ metadata={"department": "engineering", "cost_center": "ai_ops"}
+ )
+ elif api_key.startswith("sk-team-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="team_user_456",
+ user_email="user@company.com",
+ user_role=LitellmUserRoles.INTERNAL_USER,
+ team_id="dev_team",
+ team_alias="Development Team",
+ max_budget=100.0,
+ tpm_limit=1000,
+ rpm_limit=20,
+ models=["gpt-3.5-turbo", "claude-3-haiku"],
+ team_member_tpm_limit=500, # Limit within team
+ end_user_tpm_limit=100, # Per end-user limit
+ metadata={"project": "chatbot_v2"}
+ )
+ else:
+ raise Exception("Invalid API key")
+ except Exception:
+ raise Exception("Authentication failed")
+```
+
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml
diff --git a/docs/my-website/docs/proxy/user_onboarding.md b/docs/my-website/docs/proxy/user_onboarding.md
new file mode 100644
index 0000000000..baa241d6cd
--- /dev/null
+++ b/docs/my-website/docs/proxy/user_onboarding.md
@@ -0,0 +1,82 @@
+# User Onboarding Guide
+
+A step-by-step guide to help admins onboard users to your LiteLLM proxy instance and help users get started with their API key.
+
+---
+
+## For Administrators
+
+### Step 1: Create a User Account
+
+You can create a user account via the Admin UI or using the API.
+
+#### Admin UI
+- Go to the (`/ui` endpoint)
+- Navigate to the Internal Users section
+- Click "Add User" and fill in the required details
+
+#### API
+```bash
+curl -X POST http://localhost:4000/user/new \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{"user_email": "user@example.com"}'
+```
+
+---
+
+### Step 2: Grant Access & Permissions
+
+- Assign the user to a team (optional)
+- Set budgets, rate limits, and allowed models as needed
+- Generate an API key for the user (via UI or API)
+
+#### **Generate API Key (API Example)**
+```bash
+curl -X POST http://localhost:4000/key/generate \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{"user_id": "", "max_budget": 100}'
+```
+
+---
+
+## For End Users
+
+### Step 3: Validate Your API Key
+
+Before making LLM calls, validate your key works by calling the `/v1/models` endpoint:
+
+```bash
+curl -X GET http://localhost:4000/v1/models \
+ -H "Authorization: Bearer "
+```
+- If your key is valid, you'll get a list of available models.
+- If invalid, you'll get a 401 error.
+
+---
+
+### Step 4: Hello World - Make Your First LLM Call
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "Hello!"}]
+ }'
+```
+
+---
+
+## Troubleshooting
+- If you get a 401 error, check with your admin that your key is active and you have access to the requested model.
+- Use the `/v1/models` endpoint to quickly check if your key is valid without consuming LLM tokens.
+
+---
+
+## See Also
+- [Proxy Quick Start](./quick_start.md)
+- [User Management](./users.md)
+- [Key Management](./key_management.md)
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index e7e0709864..a131e5c34e 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -75,6 +75,7 @@ const sidebars = {
type: "category",
label: "AI Tools (OpenWebUI, Claude Code, etc.)",
items: [
+ "integrations/letta",
"tutorials/openweb_ui",
"tutorials/openai_codex",
"tutorials/litellm_gemini_cli",
@@ -111,6 +112,7 @@ const sidebars = {
label: "Setup & Deployment",
items: [
"proxy/quick_start",
+ "proxy/user_onboarding",
"proxy/deploy",
"proxy/prod",
"proxy/cli",
@@ -132,7 +134,6 @@ const sidebars = {
label: "All Endpoints (Swagger)",
href: "https://litellm-api.up.railway.app/",
},
- "proxy/enterprise",
"proxy/management_cli",
{
type: "category",
@@ -154,11 +155,9 @@ const sidebars = {
"proxy/token_auth",
"proxy/service_accounts",
"proxy/access_control",
- "proxy/cli_sso",
- "proxy/custom_auth",
"proxy/ip_address",
"proxy/email",
- "proxy/multiple_admins",
+ "proxy/custom_auth",
],
},
{
@@ -169,30 +168,6 @@ const sidebars = {
"proxy/team_model_add"
]
},
- {
- type: "category",
- label: "Admin UI",
- items: [
- "proxy/ui",
- "proxy/admin_ui_sso",
- "proxy/custom_root_ui",
- "proxy/model_hub",
- "proxy/self_serve",
- "proxy/public_teams",
- "tutorials/scim_litellm",
- "proxy/custom_sso",
- "proxy/ui_credentials",
- "proxy/ui/bulk_edit_users",
- {
- type: "category",
- label: "UI Logs",
- items: [
- "proxy/ui_logs",
- "proxy/ui_logs_sessions"
- ]
- }
- ],
- },
{
type: "category",
label: "Spend Tracking",
@@ -203,6 +178,47 @@ const sidebars = {
label: "Budgets + Rate Limits",
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/dynamic_rate_limit", "proxy/customers"],
},
+ {
+ type: "category",
+ label: "Enterprise Features",
+ items: [
+ "proxy/enterprise",
+ {
+ type: "category",
+ label: "Admin UI",
+ items: [
+ "proxy/ui",
+ "proxy/admin_ui_sso",
+ "proxy/custom_root_ui",
+ "proxy/model_hub",
+ "proxy/self_serve",
+ "proxy/public_teams",
+ "proxy/ui_credentials",
+ "proxy/ui/bulk_edit_users",
+ {
+ type: "category",
+ label: "UI Logs",
+ items: [
+ "proxy/ui_logs",
+ "proxy/ui_logs_sessions"
+ ]
+ }
+ ],
+ },
+ {
+ type: "category",
+ label: "SSO & Identity Management",
+ items: [
+ "proxy/cli_sso",
+ "proxy/admin_ui_sso",
+ "proxy/custom_sso",
+ "tutorials/scim_litellm",
+ "tutorials/msft_sso",
+ "proxy/multiple_admins",
+ ],
+ },
+ ],
+ },
{
type: "link",
label: "Load Balancing, Routing, Fallbacks",
@@ -250,6 +266,25 @@ const sidebars = {
slug: "/supported_endpoints",
},
items: [
+ "anthropic_unified",
+ "apply_guardrail",
+ "assistants",
+ {
+ type: "category",
+ label: "/audio",
+ "items": [
+ "audio_transcription",
+ "text_to_speech",
+ ]
+ },
+ {
+ type: "category",
+ label: "/batches",
+ items: [
+ "batches",
+ "proxy/managed_batches",
+ ]
+ },
{
type: "category",
label: "/chat/completions",
@@ -266,11 +301,23 @@ const sidebars = {
"completion/http_handler_config",
],
},
- "response_api",
- "text_completion",
"embedding/supported_embedding",
- "anthropic_unified",
- "mcp",
+ {
+ type: "category",
+ label: "/files",
+ items: [
+ "files_endpoints",
+ "proxy/litellm_managed_files",
+ ],
+ },
+ {
+ type: "category",
+ label: "/fine_tuning",
+ items: [
+ "fine_tuning",
+ "proxy/managed_finetuning",
+ ]
+ },
"generateContent",
{
type: "category",
@@ -281,21 +328,8 @@ const sidebars = {
"image_variations",
]
},
- {
- type: "category",
- label: "/audio",
- "items": [
- "audio_transcription",
- "text_to_speech",
- ]
- },
- {
- type: "category",
- label: "/vector_stores",
- items: [
- "vector_stores/search",
- ]
- },
+ "mcp",
+ "moderation",
{
type: "category",
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
@@ -314,36 +348,17 @@ const sidebars = {
"proxy/pass_through",
],
},
- "rerank",
- "assistants",
-
- {
- type: "category",
- label: "/files",
- items: [
- "files_endpoints",
- "proxy/litellm_managed_files",
- ],
- },
- {
- type: "category",
- label: "/batches",
- items: [
- "batches",
- "proxy/managed_batches",
- ]
- },
"realtime",
+ "rerank",
+ "response_api",
+ "text_completion",
{
type: "category",
- label: "/fine_tuning",
+ label: "/vector_stores",
items: [
- "fine_tuning",
- "proxy/managed_finetuning",
+ "vector_stores/search",
]
},
- "moderation",
- "apply_guardrail",
],
},
{