From 476f0b29d2e33e72891801a7115161cf0672bee2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 29 Jan 2026 17:48:38 -0800 Subject: [PATCH] [Feat] LiteLLM x Claude Agent SDK Integration (#20035) * fix: bedrock invoke - does not support prompt-caching-scope * fix: UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS * init requirements.txt * init README for claude Agent SDK * fix: using converse models with UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS * fix main.py * init: proxy_e2e_anthropic_messages_tests --- .circleci/config.yml | 112 ++++++++++ cookbook/anthropic_agent_sdk/README.md | 117 +++++++++++ .../anthropic_agent_sdk/config.example.yaml | 25 +++ cookbook/anthropic_agent_sdk/main.py | 196 ++++++++++++++++++ cookbook/anthropic_agent_sdk/requirements.txt | 2 + .../bedrock/chat/converse_transformation.py | 47 ++++- .../anthropic_claude3_transformation.py | 1 + litellm/proxy/proxy_config.yaml | 114 ++-------- .../test_claude_agent_sdk.py | 120 +++++++++++ .../test_config.yaml | 31 +++ 10 files changed, 669 insertions(+), 96 deletions(-) create mode 100644 cookbook/anthropic_agent_sdk/README.md create mode 100644 cookbook/anthropic_agent_sdk/config.example.yaml create mode 100644 cookbook/anthropic_agent_sdk/main.py create mode 100644 cookbook/anthropic_agent_sdk/requirements.txt create mode 100644 tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py create mode 100644 tests/proxy_e2e_anthropic_messages_tests/test_config.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 84b15572a6..e5bc82a596 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3407,6 +3407,110 @@ jobs: - store_test_results: path: test-results + proxy_e2e_anthropic_messages_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.10 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.10 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install "boto3==1.36.0" + pip install "httpx==0.27.0" + pip install "claude-agent-sdk" + pip install -r requirements.txt + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container with test config + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e AWS_REGION_NAME="us-east-1" \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run Claude Agent SDK E2E Tests + command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + export LITELLM_PROXY_URL="http://localhost:4000" + export LITELLM_API_KEY="sk-1234" + pwd + ls + python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + + # Store test results + - store_test_results: + path: test-results + upload-coverage: docker: - image: cimg/python:3.9 @@ -4075,6 +4179,14 @@ workflows: only: - main - /litellm_.*/ + - proxy_e2e_anthropic_messages_tests: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ - llm_translation_testing: filters: branches: diff --git a/cookbook/anthropic_agent_sdk/README.md b/cookbook/anthropic_agent_sdk/README.md new file mode 100644 index 0000000000..f113261809 --- /dev/null +++ b/cookbook/anthropic_agent_sdk/README.md @@ -0,0 +1,117 @@ +# Claude Agent SDK with LiteLLM Gateway + +A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK. + +## Quick Start + +### 1. Install dependencies + +```bash +pip install anthropic claude-agent-sdk litellm +``` + +### 2. Start LiteLLM proxy + +```bash +# Simple start with Claude +litellm --model claude-sonnet-4-20250514 + +# Or with a config file +litellm --config config.yaml +``` + +### 3. Run the chat + +```bash +python main.py +``` + +That's it! You can now chat with the agent in your terminal. + +### Chat Commands + +While chatting, you can use these commands: +- `models` - List all available models (fetched from your LiteLLM proxy) +- `model` - Switch to a different model +- `clear` - Start a new conversation +- `quit` or `exit` - End the chat + +The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured. + +## Configuration + +Set these environment variables if needed: + +```bash +export LITELLM_PROXY_URL="http://localhost:4000" +export LITELLM_API_KEY="sk-1234" +export LITELLM_MODEL="claude-sonnet-4-20250514" +``` + +Or just use the defaults - it'll connect to `http://localhost:4000` by default. + +## Example Config File + +If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`): + +```yaml +model_list: + - model_name: bedrock-claude-sonnet-4 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1" +``` + +Then start LiteLLM with: `litellm --config config.yaml` + +## How It Works + +The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic: + +```python +# Point to LiteLLM gateway (not Anthropic) +os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" +os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key + +# Use any model configured in LiteLLM +options = ClaudeAgentOptions( + model="bedrock-claude-sonnet-4", # or gpt-4, or anything else + system_prompt="You are a helpful assistant.", + max_turns=50, +) +``` + +Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically. + +## Why Use This? + +- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc. +- **Cost tracking**: LiteLLM tracks spending across all your agent conversations +- **Rate limiting**: Set budgets and limits on your agent usage +- **Load balancing**: Distribute requests across multiple API keys or regions +- **Fallbacks**: Automatically retry with a different model if one fails + +## Troubleshooting + +**Connection errors?** +- Make sure LiteLLM is running: `litellm --model your-model` +- Check the URL is correct (default: `http://localhost:4000`) + +**Authentication errors?** +- Verify your LiteLLM API key is correct +- Make sure the model is configured in your LiteLLM setup + +**Model not found?** +- Check the model name matches what's in your LiteLLM config +- Run `litellm --model your-model` to test it works + +## Learn More + +- [LiteLLM Docs](https://docs.litellm.ai/) +- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk) +- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start) diff --git a/cookbook/anthropic_agent_sdk/config.example.yaml b/cookbook/anthropic_agent_sdk/config.example.yaml new file mode 100644 index 0000000000..eb1984fc4e --- /dev/null +++ b/cookbook/anthropic_agent_sdk/config.example.yaml @@ -0,0 +1,25 @@ +model_list: + - model_name: bedrock-claude-sonnet-3.5 + litellm_params: + model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-opus-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-nova-premier + litellm_params: + model: "bedrock/amazon.nova-premier-v1:0" + aws_region_name: "us-east-1" diff --git a/cookbook/anthropic_agent_sdk/main.py b/cookbook/anthropic_agent_sdk/main.py new file mode 100644 index 0000000000..9bdd2f7364 --- /dev/null +++ b/cookbook/anthropic_agent_sdk/main.py @@ -0,0 +1,196 @@ +""" +Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway + +This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy. +LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.) +through the Claude Agent SDK by pointing it to the LiteLLM gateway. +""" + +import os +import asyncio +import httpx +from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions + + +class Config: + """Configuration for LiteLLM Gateway connection""" + + # LiteLLM proxy URL (default to local instance) + LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") + + # LiteLLM API key (master key or virtual key) + LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234") + + # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.) + LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5") + + +async def fetch_available_models(base_url: str, api_key: str) -> list[str]: + """ + Fetch available models from LiteLLM proxy /models endpoint + """ + try: + async with httpx.AsyncClient() as client: + response = await client.get( + f"{base_url}/models", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=10.0 + ) + response.raise_for_status() + data = response.json() + return [model["id"] for model in data.get("data", [])] + except Exception as e: + print(f"āš ļø Warning: Could not fetch models from proxy: {e}") + print("Using default model list...") + # Fallback to default models + return [ + "bedrock-claude-sonnet-3.5", + "bedrock-claude-sonnet-4", + "bedrock-claude-sonnet-4.5", + "bedrock-claude-opus-4.5", + "bedrock-nova-premier", + ] + + +async def interactive_chat(): + """ + Interactive CLI chat with the agent + """ + config = Config() + + # Configure Anthropic SDK to point to LiteLLM gateway + # Note: We don't add /anthropic to the base URL - LiteLLM handles routing + litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/') + os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url + os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY + + # Fetch available models from proxy + available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY) + + current_model = config.LITELLM_MODEL + + print("=" * 70) + print("šŸ¤– Claude Agent SDK with LiteLLM Gateway - Interactive Chat") + print("=" * 70) + print(f"šŸš€ Connected to: {litellm_base_url}") + print(f"šŸ“¦ Current model: {current_model}") + print("\nType your messages below. Commands:") + print(" - 'quit' or 'exit' to end the conversation") + print(" - 'clear' to start a new conversation") + print(" - 'model' to switch models") + print(" - 'models' to list available models") + print("=" * 70) + print() + + while True: + # Configure agent options for each conversation + options = ClaudeAgentOptions( + system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", + model=current_model, + max_turns=50, + ) + + # Create agent client + async with ClaudeSDKClient(options=options) as client: + conversation_active = True + + while conversation_active: + # Get user input + try: + user_input = input("\nšŸ‘¤ You: ").strip() + except (EOFError, KeyboardInterrupt): + print("\n\nšŸ‘‹ Goodbye!") + return + + # Handle commands + if user_input.lower() in ['quit', 'exit']: + print("\nšŸ‘‹ Goodbye!") + return + + if user_input.lower() == 'clear': + print("\nšŸ”„ Starting new conversation...\n") + conversation_active = False + continue + + if user_input.lower() == 'models': + print("\nšŸ“‹ Available models:") + for i, model in enumerate(available_models, 1): + marker = "āœ“" if model == current_model else " " + print(f" {marker} {i}. {model}") + continue + + if user_input.lower() == 'model': + print("\nšŸ“‹ Select a model:") + for i, model in enumerate(available_models, 1): + marker = "āœ“" if model == current_model else " " + print(f" {marker} {i}. {model}") + + try: + choice = input("\nEnter number (or press Enter to cancel): ").strip() + if choice: + idx = int(choice) - 1 + if 0 <= idx < len(available_models): + current_model = available_models[idx] + print(f"\nāœ… Switched to: {current_model}") + print("šŸ”„ Starting new conversation with new model...\n") + conversation_active = False + else: + print("āŒ Invalid choice") + except (ValueError, IndexError): + print("āŒ Invalid input") + continue + + if not user_input: + continue + + # Send query to agent with loading indicator + print("\nšŸ¤– Assistant: ", end='', flush=True) + + try: + await client.query(user_input) + + # Show loading indicator + print("ā³ thinking...", end='', flush=True) + + # Stream the response + first_chunk = True + async for msg in client.receive_response(): + # Clear loading indicator on first message + if first_chunk: + print("\ršŸ¤– Assistant: ", end='', flush=True) + first_chunk = False + + # Handle different message types + if hasattr(msg, 'type'): + if msg.type == 'content_block_delta': + # Streaming text delta + if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'): + print(msg.delta.text, end='', flush=True) + elif msg.type == 'content_block_start': + # Start of content block + if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'): + print(msg.content_block.text, end='', flush=True) + + # Fallback to original content handling + if hasattr(msg, 'content'): + for content_block in msg.content: + if hasattr(content_block, 'text'): + print(content_block.text, end='', flush=True) + + print() # New line after response + + except Exception as e: + print(f"\r\nāŒ Error: {e}") + print("Please check your LiteLLM gateway is running and configured correctly.") + + +def main(): + """Run interactive chat""" + try: + asyncio.run(interactive_chat()) + except KeyboardInterrupt: + print("\n\nšŸ‘‹ Goodbye!") + + +if __name__ == "__main__": + main() diff --git a/cookbook/anthropic_agent_sdk/requirements.txt b/cookbook/anthropic_agent_sdk/requirements.txt new file mode 100644 index 0000000000..1e810bb7d9 --- /dev/null +++ b/cookbook/anthropic_agent_sdk/requirements.txt @@ -0,0 +1,2 @@ +claude-agent-sdk +httpx>=0.27.0 diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index ec66514207..0d29c1f01a 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -76,6 +76,13 @@ BEDROCK_COMPUTER_USE_TOOLS = [ "text_editor_", ] +# Beta header patterns that are not supported by Bedrock Converse API +# These will be filtered out to prevent errors +UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS = [ + "advanced-tool-use", # Bedrock Converse doesn't support advanced-tool-use beta headers + "prompt-caching", # Prompt caching not supported in Converse API +] + class AmazonConverseConfig(BaseConfig): """ @@ -610,6 +617,37 @@ class AmazonConverseConfig(BaseConfig): return transformed_tools + def _filter_unsupported_beta_headers_for_bedrock( + self, model: str, beta_list: list + ) -> list: + """ + Remove beta headers that are not supported on Bedrock Converse API for the given model. + + Extended thinking beta headers are only supported on specific Claude 4+ models. + Some beta headers are universally unsupported on Bedrock Converse API. + + Args: + model: The model name + beta_list: The list of beta headers to filter + + Returns: + Filtered list of beta headers + """ + filtered_betas = [] + + # 1. Filter out beta headers that are universally unsupported on Bedrock Converse + for beta in beta_list: + should_keep = True + for unsupported_pattern in UNSUPPORTED_BEDROCK_CONVERSE_BETA_PATTERNS: + if unsupported_pattern in beta.lower(): + should_keep = False + break + + if should_keep: + filtered_betas.append(beta) + + return filtered_betas + def _separate_computer_use_tools( self, tools: List[OpenAIChatCompletionToolParam], model: str ) -> Tuple[ @@ -1088,7 +1126,14 @@ class AmazonConverseConfig(BaseConfig): if beta not in seen: unique_betas.append(beta) seen.add(beta) - additional_request_params["anthropic_beta"] = unique_betas + + # Filter out unsupported beta headers for Bedrock Converse API + filtered_betas = self._filter_unsupported_beta_headers_for_bedrock( + model=model, + beta_list=unique_betas, + ) + + additional_request_params["anthropic_beta"] = filtered_betas return bedrock_tools, anthropic_beta_list diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index fc4220c054..b1c45ea83a 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -54,6 +54,7 @@ class AmazonAnthropicClaudeMessagesConfig( # These will be filtered out to prevent 400 "invalid beta flag" errors UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [ "advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers + "prompt-caching-scope" ] def __init__(self, **kwargs): diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index ea405c1dea..e12e75b54f 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,101 +1,25 @@ model_list: - - model_name: "*" + - model_name: bedrock-claude-sonnet-3.5 litellm_params: - model: "*" - - model_name: "gpt-4" + model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4 litellm_params: - model: "gpt-4" - api_key: os.environ/OPENAI_API_KEY - - model_name: "gpt-3.5-turbo" + model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4.5 litellm_params: - model: "gpt-3.5-turbo" - api_key: os.environ/OPENAI_API_KEY + model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1" -general_settings: - master_key: sk-1234 + - model_name: bedrock-claude-opus-4.5 + litellm_params: + model: "bedrock/converse/us.anthropic.claude-opus-4-5-20251101-v1:0" + aws_region_name: "us-east-1" -# ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -# POLICIES - Define WHAT guardrails to apply -# ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -# -# Policies define guardrails with: -# - inherit: Inherit guardrails from another policy -# - description: Human-readable description -# - guardrails.add: Add guardrails (on top of inherited) -# - guardrails.remove: Remove guardrails (from inherited) -# - condition.model: Model pattern (exact or regex) for when guardrails apply -# -policies: - # Global baseline policy - global-baseline: - description: "Base guardrails for all requests" - guardrails: - add: - - pii_blocker - - # Healthcare policy - inherits from global-baseline - healthcare-compliance: - inherit: global-baseline - description: "HIPAA compliance for healthcare teams" - guardrails: - add: - - hipaa_audit - - # Dev policy - inherits but removes PII blocker for testing - internal-dev: - inherit: global-baseline - description: "Relaxed policy for internal development" - guardrails: - add: - - toxicity_filter - remove: - - pii_blocker - - # Policy with model condition (regex pattern) - gpt4-safety: - description: "Extra safety for GPT-4 models" - guardrails: - add: - - toxicity_filter - condition: - model: "gpt-4.*" # regex: matches gpt-4, gpt-4-turbo, gpt-4o, etc. - - # Policy with model condition (exact match list) - bedrock-compliance: - description: "Compliance for Bedrock models" - guardrails: - add: - - strict_pii_blocker - condition: - model: ["bedrock/claude-3", "bedrock/claude-2"] # exact matches - -# ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -# POLICY ATTACHMENTS - Define WHERE policies apply -# ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ -# -# Attachments are REQUIRED to make policies active. -# A policy without an attachment will not be applied. -# -policy_attachments: - # Global attachment - applies to all requests - - policy: global-baseline - scope: "*" - - # Team-specific attachment - - policy: healthcare-compliance - teams: - - healthcare-team - - medical-research - - # Key pattern attachment - - policy: internal-dev - keys: - - "dev-key-*" - - "test-key-*" - - # Model-specific policies (attached globally, condition filters by model) - - policy: gpt4-safety - scope: "*" - - - policy: bedrock-compliance - scope: "*" + - model_name: bedrock-nova-premier + litellm_params: + model: "bedrock/us.amazon.nova-premier-v1:0" + aws_region_name: "us-east-1" \ No newline at end of file diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py new file mode 100644 index 0000000000..0838faf421 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py @@ -0,0 +1,120 @@ +""" +E2E tests for Claude Agent SDK with LiteLLM Proxy using Bedrock models. + +Tests streaming messages across different Bedrock models: +- Regular Bedrock Claude Sonnet 4.5 +- Bedrock Converse Claude Sonnet 4.5 +- AWS Nova Premier +""" + +import os +import pytest +import asyncio +from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions + + +# Test models from proxy_config.yaml +TEST_MODELS = [ + ("bedrock-claude-sonnet-4.5", "Bedrock Invoke API"), + ("bedrock-converse-claude-sonnet-4.5", "Bedrock Converse API"), + ("bedrock-nova-premier", "AWS Nova Premier"), +] + + +@pytest.fixture(scope="module") +def litellm_proxy_config(): + """Configure connection to LiteLLM proxy""" + proxy_url = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") + api_key = os.getenv("LITELLM_API_KEY", "sk-1234") + + # Set environment variables for Claude Agent SDK + os.environ["ANTHROPIC_BASE_URL"] = proxy_url.rstrip('/') + os.environ["ANTHROPIC_API_KEY"] = api_key + + return { + "proxy_url": proxy_url, + "api_key": api_key, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name,model_description", TEST_MODELS) +async def test_claude_agent_sdk_streaming(litellm_proxy_config, model_name, model_description): + """ + Test streaming messages with Claude Agent SDK through LiteLLM proxy. + + This validates: + 1. Claude Agent SDK can connect to LiteLLM proxy + 2. Streaming works correctly + 3. Different Bedrock models (Invoke, Converse, Nova) work end-to-end + """ + print(f"\n{'='*60}") + print(f"Testing: {model_name} ({model_description})") + print(f"{'='*60}") + + # Configure agent options + options = ClaudeAgentOptions( + system_prompt="You are a helpful AI assistant. Be concise.", + model=model_name, + max_turns=5, + ) + + # Test query + test_query = "Say 'Hello from LiteLLM!' and nothing else." + + # Track streaming + received_chunks = [] + full_response = "" + + try: + async with ClaudeSDKClient(options=options) as client: + await client.query(test_query) + + # Collect streaming response + async for msg in client.receive_response(): + # Handle different message types + if hasattr(msg, 'type'): + if msg.type == 'content_block_delta': + # Streaming text delta + if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'): + chunk_text = msg.delta.text + received_chunks.append(chunk_text) + full_response += chunk_text + elif msg.type == 'content_block_start': + # Start of content block + if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'): + chunk_text = msg.content_block.text + received_chunks.append(chunk_text) + full_response += chunk_text + + # Fallback to content handling + if hasattr(msg, 'content'): + for content_block in msg.content: + if hasattr(content_block, 'text'): + chunk_text = content_block.text + received_chunks.append(chunk_text) + full_response += chunk_text + + # Assertions + print(f"\nāœ… Received {len(received_chunks)} chunks") + print(f"šŸ“ Full response: {full_response[:100]}...") + + # Verify we got a response + assert len(full_response) > 0, f"No response received from {model_name}" + + # Verify streaming (should have multiple chunks for most responses) + # Note: Very short responses might come in 1 chunk, so we just verify we got content + assert len(received_chunks) > 0, f"No chunks received from {model_name}" + + # Verify response contains expected content (case insensitive) + assert "hello" in full_response.lower(), f"Response doesn't contain expected greeting: {full_response}" + + print(f"āœ… Test passed for {model_name}") + + except Exception as e: + pytest.fail(f"Test failed for {model_name} ({model_description}): {str(e)}") + + +if __name__ == "__main__": + # Run tests + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml new file mode 100644 index 0000000000..35367860f1 --- /dev/null +++ b/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml @@ -0,0 +1,31 @@ +model_list: + - model_name: bedrock-claude-sonnet-3.5 + litellm_params: + model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-sonnet-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-claude-opus-4.5 + litellm_params: + model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0" + aws_region_name: "us-east-1" + + - model_name: bedrock-nova-premier + litellm_params: + model: "bedrock/amazon.nova-premier-v1:0" + aws_region_name: "us-east-1" + + # Converse API models + - model_name: bedrock-converse-claude-sonnet-4.5 + litellm_params: + model: "bedrock_converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0" + aws_region_name: "us-east-1"