diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 00fe45e99e..52d14ee941 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -377,6 +377,112 @@ print(f"\nResponse: {resp}") ``` +## Usage - 'thinking' / 'reasoning content' + +This is currently only supported for Anthropic's Claude 3.5 Sonnet. + +Works for: +- sync completion calls (SDK) - v1.61.19+ +- async completion calls (SDK + PROXY) - v1.61.20+ + + + + +```python +from litellm import completion + +# set env +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + + +resp = completion( + model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 1024}, +) + +print(resp) +``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: bedrock-claude-3-7 + litellm_params: + model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 + thinking: {"type": "enabled", "budget_tokens": 1024} # 👈 EITHER HERE OR ON REQUEST +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "bedrock-claude-3-7", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "thinking": {"type": "enabled", "budget_tokens": 1024} # 👈 EITHER HERE OR ON CONFIG.YAML + }' +``` + + + + + +**Expected Response** + +Same as Anthropic API response. + +```python +{ + "id": "chatcmpl-c661dfd7-7530-49c9-b0cc-d5018ba4727d", + "created": 1740640366, + "model": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", + "object": "chat.completion", + "system_fingerprint": null, + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "The capital of France is Paris. It's not only the capital city but also the largest city in France, serving as the country's major cultural, economic, and political center.", + "role": "assistant", + "tool_calls": null, + "function_call": null, + "reasoning_content": "The capital of France is Paris. This is a straightforward factual question.", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The capital of France is Paris. This is a straightforward factual question.", + "signature_delta": "EqoBCkgIARABGAIiQL2UoU0b1OHYi+yCHpBY7U6FQW8/FcoLewocJQPa2HnmLM+NECy50y44F/kD4SULFXi57buI9fAvyBwtyjlOiO0SDE3+r3spdg6PLOo9PBoMma2ku5OTAoR46j9VIjDRlvNmBvff7YW4WI9oU8XagaOBSxLPxElrhyuxppEn7m6bfT40dqBSTDrfiw4FYB4qEPETTI6TA6wtjGAAqmFqKTo=" + } + ] + } + } + ], + "usage": { + "completion_tokens": 64, + "prompt_tokens": 42, + "total_tokens": 106, + "completion_tokens_details": null, + "prompt_tokens_details": null + } +} +``` + + ## Usage - Bedrock Guardrails Example of using [Bedrock Guardrails with LiteLLM](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-use-converse-api.html) diff --git a/litellm/main.py b/litellm/main.py index b19c4b31d5..57bcda61fd 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -433,6 +433,7 @@ async def acompletion( "reasoning_effort": reasoning_effort, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter + "thinking": thinking, } if custom_llm_provider is None: _, custom_llm_provider, _, _ = get_llm_provider( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 68987ea1e7..d9180abd4e 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -14,6 +14,9 @@ model_list: litellm_params: model: bedrock/cohere.embed-english-v3 api_key: os.environ/COHERE_API_KEY + - model_name: bedrock-claude-3-7 + litellm_params: + model: bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0 litellm_settings: callbacks: ["langfuse"] \ No newline at end of file