From c5b51cd2b4cc905759f5f841983cc16f109297a9 Mon Sep 17 00:00:00 2001 From: Cole McIntosh <82463175+colesmcintosh@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:52:45 -0600 Subject: [PATCH 1/7] feat: add Morph provider support (#12821) * feat: add Morph provider support - Add MorphChatConfig implementation for OpenAI-compatible API - Support morph-v3-fast and morph-v3-large models - Add pricing: morph-v3-fast (/bin/zsh.8/.2 per 1M tokens), morph-v3-large (/bin/zsh.9/.9 per 1M tokens) - Both models support 16k context window and system messages - Add comprehensive documentation and unit tests - Update all necessary integration points (constants, init, provider logic) * feat: Add Morph provider support in ProviderConfigManager - Extend ProviderConfigManager to include MorphChatConfig for the Morph LLM provider. - Update MorphChatConfig by removing unused parameters from the configuration. --- docs/my-website/docs/providers/morph.md | 123 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 6 + litellm/constants.py | 3 + .../get_llm_provider_logic.py | 7 + litellm/llms/morph/__init__.py | 0 litellm/llms/morph/chat/__init__.py | 0 litellm/llms/morph/chat/transformation.py | 46 +++++++ ...odel_prices_and_context_window_backup.json | 28 ++++ litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 28 ++++ tests/llm_translation/test_morph.py | 108 +++++++++++++++ 13 files changed, 353 insertions(+) create mode 100644 docs/my-website/docs/providers/morph.md create mode 100644 litellm/llms/morph/__init__.py create mode 100644 litellm/llms/morph/chat/__init__.py create mode 100644 litellm/llms/morph/chat/transformation.py create mode 100644 tests/llm_translation/test_morph.py diff --git a/docs/my-website/docs/providers/morph.md b/docs/my-website/docs/providers/morph.md new file mode 100644 index 0000000000..e49c60b566 --- /dev/null +++ b/docs/my-website/docs/providers/morph.md @@ -0,0 +1,123 @@ +# Morph + +LiteLLM supports all models on [Morph](https://morphllm.com) + +## Overview + +Morph provides specialized AI models designed for agentic workflows, particularly excelling at precise code editing and manipulation. Their "Apply" models enable targeted code changes without full file rewrites, making them ideal for AI agents that need to make intelligent, context-aware code modifications. + +## API Key +```python +import os +os.environ["MORPH_API_KEY"] = "your-api-key" +``` + +## Sample Usage + +```python +from litellm import completion + +# set env variable +os.environ["MORPH_API_KEY"] = "your-api-key" + +messages = [ + {"role": "user", "content": "Write a Python function to calculate factorial"} +] + +## Morph v3 Fast - Optimized for speed +response = completion( + model="morph/morph-v3-fast", + messages=messages, +) +print(response) + +## Morph v3 Large - Most capable model +response = completion( + model="morph/morph-v3-large", + messages=messages, +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion + +# set env variable +os.environ["MORPH_API_KEY"] = "your-api-key" + +messages = [ + {"role": "user", "content": "Write a Python function to calculate factorial"} +] + +## Morph v3 Fast with streaming +response = completion( + model="morph/morph-v3-fast", + messages=messages, + stream=True, +) + +for chunk in response: + print(chunk) +``` + +## Supported Models + +| Model Name | Function Call | Description | Context Window | +|--------------------------|--------------------------------------------|-----------------------|----------------| +| morph-v3-fast | `completion('morph/morph-v3-fast', messages)` | Fastest model, optimized for quick responses | 16k tokens | +| morph-v3-large | `completion('morph/morph-v3-large', messages)` | Most capable model for complex tasks | 16k tokens | + +## Usage - LiteLLM Proxy Server + +Here's how to use Morph with the LiteLLM Proxy Server: + +1. Save API key in your environment +```bash +export MORPH_API_KEY="your-api-key" +``` + +2. Add model to config.yaml +```yaml +model_list: + - model_name: morph-v3-fast + litellm_params: + model: morph/morph-v3-fast + + - model_name: morph-v3-large + litellm_params: + model: morph/morph-v3-large +``` + +3. Start the proxy server +```bash +litellm --config config.yaml +``` + +## Advanced Usage + +### Setting API Base +```python +import litellm + +# set custom api base +response = completion( + model="morph/morph-v3-large", + messages=[{"role": "user", "content": "Hello, world!"}], + api_base="https://api.morphllm.com/v1" +) +print(response) +``` + +### Setting API Key +```python +import litellm + +# set api key via completion +response = completion( + model="morph/morph-v3-large", + messages=[{"role": "user", "content": "Hello, world!"}], + api_key="your-api-key" +) +print(response) +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b770ac625a..8c210d437b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -447,6 +447,7 @@ const sidebars = { "providers/replicate", "providers/togetherai", "providers/v0", + "providers/morph", "providers/lambda_ai", "providers/novita", "providers/voyage", diff --git a/litellm/__init__.py b/litellm/__init__.py index db77eca6c3..1f08a6c03a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -503,6 +503,7 @@ elevenlabs_models: List = [] dashscope_models: List = [] moonshot_models: List = [] v0_models: List = [] +morph_models: List = [] lambda_ai_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: @@ -684,6 +685,8 @@ def add_known_models(): moonshot_models.append(key) elif value.get("litellm_provider") == "v0": v0_models.append(key) + elif value.get("litellm_provider") == "morph": + morph_models.append(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.append(key) @@ -771,6 +774,7 @@ model_list = ( + dashscope_models + moonshot_models + v0_models + + morph_models + lambda_ai_models ) @@ -840,6 +844,7 @@ models_by_provider: dict = { "dashscope": dashscope_models, "moonshot": moonshot_models, "v0": v0_models, + "morph": morph_models, "lambda_ai": lambda_ai_models, } @@ -1161,6 +1166,7 @@ from .llms.nebius.chat.transformation import NebiusConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig from .llms.v0.chat.transformation import V0ChatConfig +from .llms.morph.chat.transformation import MorphChatConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig from .main import * # type: ignore from .integrations import * diff --git a/litellm/constants.py b/litellm/constants.py index d7d3874882..3089ae131d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -279,6 +279,7 @@ LITELLM_CHAT_PROVIDERS = [ "dashscope", "moonshot", "v0", + "morph", "lambda_ai", ] @@ -409,6 +410,7 @@ openai_compatible_endpoints: List = [ "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "https://api.moonshot.ai/v1", "https://api.v0.dev/v1", + "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", ] @@ -448,6 +450,7 @@ openai_compatible_providers: List = [ "dashscope", "moonshot", "v0", + "morph", "lambda_ai", ] openai_text_completion_compatible_providers: List = ( diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 2b5ee4b18c..32d837c1e0 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -694,6 +694,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.V0ChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "morph": + ( + api_base, + dynamic_api_key, + ) = litellm.MorphChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "lambda_ai": ( api_base, diff --git a/litellm/llms/morph/__init__.py b/litellm/llms/morph/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/morph/chat/__init__.py b/litellm/llms/morph/chat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/morph/chat/transformation.py b/litellm/llms/morph/chat/transformation.py new file mode 100644 index 0000000000..f37ed8e45e --- /dev/null +++ b/litellm/llms/morph/chat/transformation.py @@ -0,0 +1,46 @@ +""" +Transform request from OpenAI format to Morph format. + +[TODO] Docs: Morph supports the OpenAI API format. +https://docs.morphllm.com/quickstart +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class MorphChatConfig(OpenAILikeChatConfig): + """ + Transform request from OpenAI format to Morph format. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "morph" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("MORPH_API_BASE") + or "https://api.morphllm.com/v1" # default api base + ) + dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + return [ + "messages", + "model", + "stream", + ] + + def pre_call(self, messages: list, model: str, api_key: str, api_base: str): + """ + Hook for any pre-processing before the API call. + """ + return \ No newline at end of file diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4920f560c8..e3257a6ebf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16889,5 +16889,33 @@ "supports_vision": true, "mode": "chat", "source": "https://platform.moonshot.ai/docs/pricing" + }, + "morph/morph-v3-fast": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "morph", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "morph/morph-v3-large": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 1.9e-06, + "litellm_provider": "morph", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_system_messages": true, + "supports_tool_choice": false } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index eba8f65c37..45714a18d0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2276,6 +2276,7 @@ class LlmProviders(str, Enum): DASHSCOPE = "dashscope" MOONSHOT = "moonshot" V0 = "v0" + MORPH = "morph" LAMBDA_AI = "lambda_ai" DEEPSEEK = "deepseek" SAMBANOVA = "sambanova" diff --git a/litellm/utils.py b/litellm/utils.py index fbfe71cf96..1556351e60 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6834,6 +6834,8 @@ class ProviderConfigManager: return litellm.MoonshotChatConfig() elif litellm.LlmProviders.V0 == provider: return litellm.V0ChatConfig() + elif litellm.LlmProviders.MORPH == provider: + return litellm.MorphChatConfig() elif litellm.LlmProviders.BEDROCK == provider: bedrock_route = BedrockModelInfo.get_bedrock_route(model) bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4920f560c8..e3257a6ebf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16889,5 +16889,33 @@ "supports_vision": true, "mode": "chat", "source": "https://platform.moonshot.ai/docs/pricing" + }, + "morph/morph-v3-fast": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 8e-07, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "morph", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "morph/morph-v3-large": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 9e-07, + "output_cost_per_token": 1.9e-06, + "litellm_provider": "morph", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false, + "supports_system_messages": true, + "supports_tool_choice": false } } diff --git a/tests/llm_translation/test_morph.py b/tests/llm_translation/test_morph.py new file mode 100644 index 0000000000..7d2568a0a7 --- /dev/null +++ b/tests/llm_translation/test_morph.py @@ -0,0 +1,108 @@ +"""Unit tests for Morph provider integration.""" + +import os +import sys +from unittest.mock import patch + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import MorphChatConfig, get_llm_provider + +# Force model loading +litellm.add_known_models() + + +def test_morph_config_get_provider_info(): + """Test that MorphChatConfig returns correct provider info.""" + config = MorphChatConfig() + + # Test with environment variable + with patch.dict(os.environ, {"MORPH_API_KEY": "test-key-from-env"}): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://api.morphllm.com/v1" + assert api_key == "test-key-from-env" + + # Test with passed api_key + api_base, api_key = config._get_openai_compatible_provider_info(None, "direct-key") + assert api_base == "https://api.morphllm.com/v1" + assert api_key == "direct-key" + + # Test with custom api_base + api_base, api_key = config._get_openai_compatible_provider_info("https://custom.morph.com", "key") + assert api_base == "https://custom.morph.com" + assert api_key == "key" + + +def test_morph_get_llm_provider(): + """Test that get_llm_provider correctly identifies morph models.""" + # Test with morph/model format + _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-large") + assert custom_llm_provider == "morph" + + _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast") + assert custom_llm_provider == "morph" + + +def test_morph_in_provider_lists(): + """Test that morph is included in all necessary provider lists.""" + import litellm + from litellm.constants import openai_compatible_providers, openai_compatible_endpoints + + # Check morph is in openai_compatible_providers + assert "morph" in openai_compatible_providers + + # Check morph endpoint is in openai_compatible_endpoints + assert "https://api.morphllm.com/v1" in openai_compatible_endpoints + + # Check morph is in provider_list + assert "morph" in litellm.provider_list + + # Check models are in model_list after initialization + assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"]) + + +def test_morph_model_info(): + """Test that morph models have correct configuration.""" + import litellm + model_info = litellm.get_model_info("morph/morph-v3-large") + + assert model_info["litellm_provider"] == "morph" + assert model_info["mode"] == "chat" + assert model_info["max_tokens"] == 16000 + assert model_info["max_input_tokens"] == 16000 + assert model_info["max_output_tokens"] == 16000 + assert model_info["input_cost_per_token"] == 9e-07 # $0.9/1M tokens + assert model_info["output_cost_per_token"] == 1.9e-06 # $1.9/1M tokens + assert model_info["supports_function_calling"] is False + assert model_info["supports_vision"] is False + assert model_info["supports_system_messages"] is True + + +def test_morph_supported_params(): + """Test that MorphChatConfig returns correct supported parameters.""" + config = MorphChatConfig() + supported_params = config.get_supported_openai_params("morph/morph-v3-large") + + expected_params = [ + "messages", + "model", + "stream", + "temperature", + "max_tokens", + "tools", + "tool_choice", + "response_format", + ] + + assert all(param in supported_params for param in expected_params) + + +def test_morph_custom_llm_provider(): + """Test that morph models are correctly identified.""" + config = MorphChatConfig() + assert config.custom_llm_provider == "morph" + + From 6a1b2323307d06a852a935207dcffd6ec2c2a212 Mon Sep 17 00:00:00 2001 From: Adam Holmberg Date: Mon, 21 Jul 2025 16:21:21 -0500 Subject: [PATCH 2/7] fix: remove deprecated groq/qwen-qwq-32b and add qwen/qwen3-32b (#12831) fixes #12825 --- docs/my-website/docs/providers/groq.md | 2 +- litellm/model_prices_and_context_window_backup.json | 10 +++++----- model_prices_and_context_window.json | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md index 5b74151d73..59668b5eb5 100644 --- a/docs/my-website/docs/providers/groq.md +++ b/docs/my-website/docs/providers/groq.md @@ -158,7 +158,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion | mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` | | gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` | | moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` | -| qwen-qwq-32b | `completion(model="groq/qwen-qwq-32b", messages)` | +| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` | ## Groq - Tool / Function Calling Example diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e3257a6ebf..92b48fba76 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5417,12 +5417,12 @@ "supports_tool_choice": true, "deprecation_date": "2025-01-06" }, - "groq/qwen-qwq-32b": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "groq/qwen/qwen3-32b": { + "max_tokens": 131000, + "max_input_tokens": 131000, + "max_output_tokens": 131000, "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 3.9e-07, + "output_cost_per_token": 5.9e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e3257a6ebf..92b48fba76 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5417,12 +5417,12 @@ "supports_tool_choice": true, "deprecation_date": "2025-01-06" }, - "groq/qwen-qwq-32b": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, + "groq/qwen/qwen3-32b": { + "max_tokens": 131000, + "max_input_tokens": 131000, + "max_output_tokens": 131000, "input_cost_per_token": 2.9e-07, - "output_cost_per_token": 3.9e-07, + "output_cost_per_token": 5.9e-07, "litellm_provider": "groq", "mode": "chat", "supports_function_calling": true, From 774af8085efc45c27526244eacd89306fa8671fc Mon Sep 17 00:00:00 2001 From: Cole McIntosh <82463175+colesmcintosh@users.noreply.github.com> Date: Mon, 21 Jul 2025 15:24:44 -0600 Subject: [PATCH 3/7] docs: add Google Cloud Model Armor guardrail documentation (#12814) - Add comprehensive documentation for Model Armor integration - Include configuration examples and parameter descriptions - Add Model Armor to sidebars navigation - Document authentication methods and error handling --- .../docs/proxy/guardrails/model_armor.md | 93 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 94 insertions(+) create mode 100644 docs/my-website/docs/proxy/guardrails/model_armor.md diff --git a/docs/my-website/docs/proxy/guardrails/model_armor.md b/docs/my-website/docs/proxy/guardrails/model_armor.md new file mode 100644 index 0000000000..a7463a8eee --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/model_armor.md @@ -0,0 +1,93 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Google Cloud Model Armor + +LiteLLM supports Google Cloud Model Armor guardrails via the [Model Armor API](https://cloud.google.com/security-command-center/docs/model-armor-overview). + + +## Supported Guardrails + +- [Model Armor Templates](https://cloud.google.com/security-command-center/docs/manage-model-armor-templates) - Content sanitization and blocking based on configured templates + +## Quick Start +### 1. Define Guardrails on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: model-armor-shield + litellm_params: + guardrail: model_armor + mode: [pre_call, post_call] # Run on both input and output + template_id: "your-template-id" # Required: Your Model Armor template ID + project_id: "your-project-id" # Your GCP project ID + location: "us-central1" # GCP location (default: us-central1) + credentials: "path/to/credentials.json" # Path to service account key + mask_request_content: true # Enable request content masking + mask_response_content: true # Enable response content masking + fail_on_error: true # Fail request if Model Armor errors (default: true) + default_on: true # Run by default for all requests +``` + +#### Supported values for `mode` + +- `pre_call` Run **before** LLM call, on **input** +- `post_call` Run **after** LLM call, on **input & output** + +### 2. Start LiteLLM Gateway + + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 3. Test request + +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hi, my email is test@example.com"} + ], + "guardrails": ["model-armor-shield"] + }' +``` + +## Supported Params + +### Common Params + +- `api_key` - str - Google Cloud service account credentials (optional if using ADC) +- `api_base` - str - Custom Model Armor API endpoint (optional) +- `default_on` - bool - Whether to run the guardrail by default. Default is `false`. +- `mode` - Union[str, list[str]] - Mode to run the guardrail. Either `pre_call` or `post_call`. Default is `pre_call`. + +### Model Armor Specific + +- `template_id` - str - The ID of your Model Armor template (required) +- `project_id` - str - Google Cloud project ID (defaults to credentials project) +- `location` - str - Google Cloud location/region. Default is `us-central1` +- `credentials` - Union[str, dict] - Path to service account JSON file or credentials dictionary +- `api_endpoint` - str - Custom API endpoint for Model Armor (optional) +- `fail_on_error` - bool - Whether to fail requests if Model Armor encounters errors. Default is `true` +- `mask_request_content` - bool - Enable masking of sensitive content in requests. Default is `false` +- `mask_response_content` - bool - Enable masking of sensitive content in responses. Default is `false` + + +## Further Reading + +- [Control Guardrails per API Key](./quick_start#-control-guardrails-per-api-key) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8c210d437b..65b0817682 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -39,6 +39,7 @@ const sidebars = { "proxy/guardrails/lasso_security", "proxy/guardrails/guardrails_ai", "proxy/guardrails/lakera_ai", + "proxy/guardrails/model_armor", "proxy/guardrails/openai_moderation", "proxy/guardrails/pangea", "proxy/guardrails/pii_masking_v2", From 2941a555a8784bcfa0e24c3336635d27b28ee28d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 21 Jul 2025 15:01:32 -0700 Subject: [PATCH 4/7] [Feat] Add Recraft Image Generation API Support - New LLM Provider (#12832) * add recraft * init RecraftImageGenerationConfig * add get_complete_url + validate_environment * add image_generation_handler in llm http clas * fixes for transform * working recraft request * fixed img gen transform * fixes for llm http handler * test: TestRecraftImageGeneration * fixes for llm_http_handler * fix RecraftImageGenerationConfig * TestRecraftImageGenerationTransformation * add recraft API * docs recraft API * fix code QA * map_openai_params * fix recraft * cost tracking for recraft/recraftv3 * fix code qa check --- docs/my-website/docs/image_generation.md | 19 ++ docs/my-website/docs/providers/recraft.md | 161 +++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 5 + litellm/images/main.py | 98 ++++++- .../image_generation/transformation.py | 28 +- litellm/llms/custom_httpx/llm_http_handler.py | 214 ++++++++++++++ .../llms/recraft/image_generation/__init__.py | 13 + .../image_generation/transformation.py | 163 +++++++++++ ...odel_prices_and_context_window_backup.json | 18 ++ litellm/types/llms/recraft.py | 17 ++ litellm/types/utils.py | 1 + litellm/utils.py | 6 + model_prices_and_context_window.json | 18 ++ .../image_gen_tests/test_image_generation.py | 5 + .../test_recraft_image_gen_transformation.py | 270 ++++++++++++++++++ 16 files changed, 1018 insertions(+), 19 deletions(-) create mode 100644 docs/my-website/docs/providers/recraft.md create mode 100644 litellm/llms/recraft/image_generation/__init__.py create mode 100644 litellm/llms/recraft/image_generation/transformation.py create mode 100644 litellm/types/llms/recraft.py create mode 100644 tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md index 33e4c7b4cc..792a21fc1a 100644 --- a/docs/my-website/docs/image_generation.md +++ b/docs/my-website/docs/image_generation.md @@ -207,7 +207,26 @@ Use this for Stable Diffusion models hosted on Xinference See Xinference usage with LiteLLM [here](./providers/xinference.md#image-generation) +## Recraft Image Generation Models +Use this for AI-powered design and image generation with Recraft + +#### Usage + +```python showLineNumbers +from litellm import image_generation +import os + +os.environ['RECRAFT_API_KEY'] = "your-api-key" + +response = image_generation( + model="recraft/recraftv3", + prompt="A beautiful sunset over a calm ocean", +) +print(response) +``` + +See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation) ## OpenAI Compatible Image Generation Models Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference diff --git a/docs/my-website/docs/providers/recraft.md b/docs/my-website/docs/providers/recraft.md new file mode 100644 index 0000000000..dd4a91ddcf --- /dev/null +++ b/docs/my-website/docs/providers/recraft.md @@ -0,0 +1,161 @@ +# Recraft +https://www.recraft.ai/ + +## Overview + +| Property | Details | +|-------|-------| +| Description | Recraft is an AI-powered design tool that generates high-quality images with precise control over style and content. | +| Provider Route on LiteLLM | `recraft/` | +| Link to Provider Doc | [Recraft ↗](https://www.recraft.ai/docs) | +| Supported Operations | [`/images/generations`](#image-generation) | + +LiteLLM supports Recraft Image Generation calls. + +## API Base, Key +```python +# env variable +os.environ['RECRAFT_API_KEY'] = "your-api-key" +os.environ['RECRAFT_API_BASE'] = "https://external.api.recraft.ai" # [optional] +``` + +## Image Generation + +### Usage - LiteLLM Python SDK + +```python showLineNumbers +from litellm import image_generation +import os + +os.environ['RECRAFT_API_KEY'] = "your-api-key" + +# recraft image generation call +response = image_generation( + model="recraft/recraftv3", + prompt="A beautiful sunset over a calm ocean", +) +print(response) +``` + +### Usage - LiteLLM Proxy Server + +#### 1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: recraft-v3 + litellm_params: + model: recraft/recraftv3 + api_key: os.environ/RECRAFT_API_KEY + model_info: + mode: image_generation + +general_settings: + master_key: sk-1234 +``` + +#### 2. Start the proxy + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +#### 3. Test it + +```bash showLineNumbers +curl --location 'http://0.0.0.0:4000/v1/images/generations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "recraft-v3", + "prompt": "A beautiful sunset over a calm ocean", +}' +``` + +### Advanced Usage - With Additional Parameters + +```python showLineNumbers +from litellm import image_generation +import os + +os.environ['RECRAFT_API_KEY'] = "your-api-key" + +response = image_generation( + model="recraft/recraftv3", + prompt="A beautiful sunset over a calm ocean", +) +print(response) +``` + +### Supported Parameters + +Recraft supports the following OpenAI-compatible parameters: + +| Parameter | Type | Description | Example | +|-----------|------|-------------|---------| +| `n` | integer | Number of images to generate (1-4) | `1` | +| `response_format` | string | Format of response (`url` or `b64_json`) | `"url"` | +| `size` | string | Image dimensions | `"1024x1024"` | +| `style` | string | Image style/artistic direction | `"realistic"` | + +### Using Non-OpenAI Parameters + +If you want to pass parameters that are not supported by OpenAI, you can pass them in your request body, LiteLLM will automatically route it to recraft. + +In this example we will pass `style_id` parameter to the recraft image generation call. + +**Usage with LiteLLM Python SDK** + +```python showLineNumbers +from litellm import image_generation +import os + +os.environ['RECRAFT_API_KEY'] = "your-api-key" + +response = image_generation( + model="recraft/recraftv3", + prompt="A beautiful sunset over a calm ocean", + style_id="your-style-id", +) +``` + +**Usage with LiteLLM Proxy Server + OpenAI Python SDK** + +```python showLineNumbers +from openai import OpenAI +import os + +os.environ['RECRAFT_API_KEY'] = "your-api-key" + +client = OpenAI(api_key=os.environ['RECRAFT_API_KEY']) + +response = client.images.generate( + model="recraft/recraftv3", + prompt="A beautiful sunset over a calm ocean", + extra_body={ + "style_id": "your-style-id", + }, +) +print(response) +``` + +### Supported Image Generation Models + +**Note: All recraft models are supported by LiteLLM** Just pass the model name with `recraft/` and litellm will route it to recraft. + +| Model Name | Function Call | +|------------|---------------| +| recraftv3 | `image_generation(model="recraft/recraftv3", prompt="...")` | +| recraftv2 | `image_generation(model="recraft/recraftv2", prompt="...")` | + +For more details on available models and features, see: https://www.recraft.ai/docs + +## API Key Setup + +Get your API key from [Recraft's website](https://www.recraft.ai/) and set it as an environment variable: + +```bash +export RECRAFT_API_KEY="your-api-key" +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 65b0817682..df7d47678f 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -445,6 +445,7 @@ const sidebars = { "providers/github_copilot", "providers/ai21", "providers/nlp_cloud", + "providers/recraft", "providers/replicate", "providers/togetherai", "providers/v0", diff --git a/litellm/__init__.py b/litellm/__init__.py index 1f08a6c03a..66850ee209 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -505,6 +505,7 @@ moonshot_models: List = [] v0_models: List = [] morph_models: List = [] lambda_ai_models: List = [] +recraft_models: List = [] def is_bedrock_pricing_only_model(key: str) -> bool: """ @@ -689,6 +690,8 @@ def add_known_models(): morph_models.append(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.append(key) + elif value.get("litellm_provider") == "recraft": + recraft_models.append(key) add_known_models() @@ -776,6 +779,7 @@ model_list = ( + v0_models + morph_models + lambda_ai_models + + recraft_models ) model_list_set = set(model_list) @@ -846,6 +850,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "recraft": recraft_models, } # mapping for those models which have larger equivalents diff --git a/litellm/images/main.py b/litellm/images/main.py index cf9ea7a662..3a675a8168 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars from functools import partial -from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast +from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast, overload import httpx @@ -14,9 +14,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.mock_functions import mock_image_generation from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.custom_llm import CustomLLM #################### Initialize provider clients #################### +llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() from litellm.main import ( azure_chat_completions, base_llm_aiohttp_handler, @@ -26,6 +28,8 @@ from litellm.main import ( openai_image_variations, vertex_image_generation, ) + +########################################### from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import ImageGenerationRequestQuality @@ -78,17 +82,20 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse: # Await normally init_response = await loop.run_in_executor(None, func_with_context) - if isinstance(init_response, dict) or isinstance( - init_response, ImageResponse - ): ## CACHING SCENARIO - if isinstance(init_response, dict): - init_response = ImageResponse(**init_response) + + response: Optional[ImageResponse] = None + if isinstance(init_response, dict): + response = ImageResponse(**init_response) + elif isinstance(init_response, ImageResponse): ## CACHING SCENARIO response = init_response elif asyncio.iscoroutine(init_response): response = await init_response # type: ignore - else: - # Call the synchronous function using run_in_executor - response = await loop.run_in_executor(None, func_with_context) + + if response is None: + raise ValueError( + "Unable to get Image Response. Please pass a valid llm_provider." + ) + return response except Exception as e: custom_llm_provider = custom_llm_provider or "openai" @@ -101,6 +108,54 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse: ) +# Overload for when aimg_generation=True (returns Coroutine) +@overload +def image_generation( + prompt: str, + model: Optional[str] = None, + n: Optional[int] = None, + quality: Optional[Union[str, ImageGenerationRequestQuality]] = None, + response_format: Optional[str] = None, + size: Optional[str] = None, + style: Optional[str] = None, + user: Optional[str] = None, + input_fidelity: Optional[str] = None, + timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider=None, + *, + aimg_generation: Literal[True], + **kwargs, +) -> Coroutine[Any, Any, ImageResponse]: + ... + + +# Overload for when aimg_generation=False or not specified (returns ImageResponse) +@overload +def image_generation( + prompt: str, + model: Optional[str] = None, + n: Optional[int] = None, + quality: Optional[Union[str, ImageGenerationRequestQuality]] = None, + response_format: Optional[str] = None, + size: Optional[str] = None, + style: Optional[str] = None, + user: Optional[str] = None, + input_fidelity: Optional[str] = None, + timeout=600, # default to 10 minutes + api_key: Optional[str] = None, + api_base: Optional[str] = None, + api_version: Optional[str] = None, + custom_llm_provider=None, + *, + aimg_generation: Literal[False] = False, + **kwargs, +) -> ImageResponse: + ... + + @client def image_generation( # noqa: PLR0915 prompt: str, @@ -118,7 +173,10 @@ def image_generation( # noqa: PLR0915 api_version: Optional[str] = None, custom_llm_provider=None, **kwargs, -) -> ImageResponse: +) -> Union[ + ImageResponse, + Coroutine[Any, Any, ImageResponse], + ]: """ Maps the https://api.openai.com/v1/images/generations endpoint. @@ -348,6 +406,26 @@ def image_generation( # noqa: PLR0915 api_base=api_base, client=client, ) + ######################################################### + # Providers using llm_http_handler + ######################################################### + elif custom_llm_provider in ( + litellm.LlmProviders.RECRAFT, + ): + if image_generation_config is None: + raise ValueError(f"image generation config is not supported for {custom_llm_provider}") + + return llm_http_handler.image_generation_handler( + model=model, + prompt=prompt, + image_generation_provider_config=image_generation_config, + image_generation_optional_request_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params_dict, + logging_obj=litellm_logging_obj, + timeout=timeout, + client=client, + ) elif ( custom_llm_provider in litellm._custom_providers ): # Assume custom LLM provider diff --git a/litellm/llms/base_llm/image_generation/transformation.py b/litellm/llms/base_llm/image_generation/transformation.py index 134c95b1c8..fc8db8c65c 100644 --- a/litellm/llms/base_llm/image_generation/transformation.py +++ b/litellm/llms/base_llm/image_generation/transformation.py @@ -3,12 +3,12 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx -from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ModelResponse +from litellm.types.utils import ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -18,12 +18,23 @@ else: LiteLLMLoggingObj = Any -class BaseImageGenerationConfig(BaseConfig, ABC): +class BaseImageGenerationConfig(ABC): @abstractmethod def get_supported_openai_params( self, model: str ) -> List[OpenAIImageGenerationOptionalParams]: pass + + @abstractmethod + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + pass + def get_complete_url( self, @@ -64,10 +75,10 @@ class BaseImageGenerationConfig(BaseConfig, ABC): headers=headers, ) - def transform_request( + def transform_image_generation_request( self, model: str, - messages: List[AllMessageValues], + prompt: str, optional_params: dict, litellm_params: dict, headers: dict, @@ -76,20 +87,19 @@ class BaseImageGenerationConfig(BaseConfig, ABC): "ImageVariationConfig implementa 'transform_request_image_variation' for image variation models" ) - def transform_response( + def transform_image_generation_response( self, model: str, raw_response: httpx.Response, - model_response: ModelResponse, + model_response: ImageResponse, logging_obj: LiteLLMLoggingObj, request_data: dict, - messages: List[AllMessageValues], optional_params: dict, litellm_params: dict, encoding: Any, api_key: Optional[str] = None, json_mode: Optional[bool] = None, - ) -> ModelResponse: + ) -> ImageResponse: raise NotImplementedError( "ImageVariationConfig implements 'transform_response_image_variation' for image variation models" ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f77f87507e..46fd866be2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -35,6 +35,9 @@ from litellm.llms.base_llm.google_genai.transformation import ( BaseGoogleGenAIGenerateContentConfig, ) from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig @@ -2370,6 +2373,7 @@ class BaseLLMHTTPHandler: BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, + BaseImageGenerationConfig, BaseVectorStoreConfig, BaseGoogleGenAIGenerateContentConfig, BaseAnthropicMessagesConfig, @@ -2657,6 +2661,216 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + def image_generation_handler( + self, + model: str, + prompt: str, + image_generation_provider_config: BaseImageGenerationConfig, + image_generation_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: Dict, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + fake_stream: bool = False, + litellm_metadata: Optional[Dict[str, Any]] = None, + ) -> Union[ + ImageResponse, + Coroutine[Any, Any, ImageResponse], + ]: + """ + Handles image generation requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_image_generation_handler( + model=model, + prompt=prompt, + image_generation_provider_config=image_generation_provider_config, + image_generation_optional_request_params=image_generation_optional_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client if isinstance(client, AsyncHTTPHandler) else None, + fake_stream=fake_stream, + litellm_metadata=litellm_metadata, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = image_generation_provider_config.validate_environment( + api_key=litellm_params.get("api_key", None), + headers=image_generation_optional_request_params.get("extra_headers", {}) or {}, + model=model, + messages=[], + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = image_generation_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base", None), + api_key=litellm_params.get("api_key", None), + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + ) + + data = image_generation_provider_config.transform_image_generation_request( + model=model, + prompt=prompt, + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=image_generation_provider_config, + ) + + model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response( + model=model, + raw_response=response, + model_response=litellm.ImageResponse(), + logging_obj=logging_obj, + request_data=data, + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + encoding=None, + ) + + return model_response + + async def async_image_generation_handler( + self, + model: str, + prompt: str, + image_generation_provider_config: BaseImageGenerationConfig, + image_generation_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: Dict, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + fake_stream: bool = False, + litellm_metadata: Optional[Dict[str, Any]] = None, + ) -> ImageResponse: + """ + Async version of the image generation handler. + Uses async HTTP client to make requests. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + + headers = image_generation_provider_config.validate_environment( + api_key=litellm_params.get("api_key", None), + headers=image_generation_optional_request_params.get("extra_headers", {}) or {}, + model=model, + messages=[], + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = image_generation_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base", None), + api_key=litellm_params.get("api_key", None), + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + ) + + data = image_generation_provider_config.transform_image_generation_request( + model=model, + prompt=prompt, + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=image_generation_provider_config, + ) + + model_response: ImageResponse = image_generation_provider_config.transform_image_generation_response( + model=model, + raw_response=response, + model_response=litellm.ImageResponse(), + logging_obj=logging_obj, + request_data=data, + optional_params=image_generation_optional_request_params, + litellm_params=dict(litellm_params), + encoding=None, + ) + + return model_response + ###### VECTOR STORE HANDLER ###### async def async_vector_store_search_handler( self, diff --git a/litellm/llms/recraft/image_generation/__init__.py b/litellm/llms/recraft/image_generation/__init__.py new file mode 100644 index 0000000000..cb8c5624db --- /dev/null +++ b/litellm/llms/recraft/image_generation/__init__.py @@ -0,0 +1,13 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .transformation import RecraftImageGenerationConfig + +__all__ = [ + "RecraftImageGenerationConfig", +] + + +def get_recraft_image_generation_config(model: str) -> BaseImageGenerationConfig: + return RecraftImageGenerationConfig() diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py new file mode 100644 index 0000000000..f632b49f3a --- /dev/null +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -0,0 +1,163 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.llms.recraft import RecraftImageGenerationRequestParams +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class RecraftImageGenerationConfig(BaseImageGenerationConfig): + DEFAULT_BASE_URL: str = "https://external.api.recraft.ai" + IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + https://www.recraft.ai/docs#generate-image + """ + return [ + "n", + "response_format", + "size", + "style" + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + optional_params[k] = non_default_params[k] + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + complete_url: str = ( + api_base + or get_secret_str("RECRAFT_API_BASE") + or self.DEFAULT_BASE_URL + ) + + complete_url = complete_url.rstrip("/") + complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" + return complete_url + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + final_api_key: Optional[str] = ( + api_key or + get_secret_str("RECRAFT_API_KEY") + ) + if not final_api_key: + raise ValueError("RECRAFT_API_KEY is not set") + + headers["Authorization"] = f"Bearer {final_api_key}" + return headers + + + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to the recraft image generation request body + + https://www.recraft.ai/docs#generate-image + """ + recratft_image_generation_request_body: RecraftImageGenerationRequestParams = RecraftImageGenerationRequestParams( + prompt=prompt, + model=model, + **optional_params, + ) + return dict(recratft_image_generation_request_body) + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the image generation response to the litellm image response + + https://www.recraft.ai/docs#generate-image + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + if not model_response.data: + model_response.data = [] + + for image_data in response_data["data"]: + model_response.data.append(ImageObject( + url=image_data.get("url", None), + b64_json=image_data.get("b64_json", None), + )) + + return model_response \ No newline at end of file diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 92b48fba76..78fd56effc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16890,6 +16890,24 @@ "mode": "chat", "source": "https://platform.moonshot.ai/docs/pricing" }, + "recraft/recraftv3": { + "mode": "image_generation", + "input_cost_per_image": 0.04, + "litellm_provider": "recraft", + "supported_endpoints": [ + "/v1/images/generations" + ], + "source": "https://www.recraft.ai/docs#pricing" + }, + "recraft/recraftv2": { + "mode": "image_generation", + "input_cost_per_image": 0.022, + "litellm_provider": "recraft", + "supported_endpoints": [ + "/v1/images/generations" + ], + "source": "https://www.recraft.ai/docs#pricing" + }, "morph/morph-v3-fast": { "max_tokens": 16000, "max_input_tokens": 16000, diff --git a/litellm/types/llms/recraft.py b/litellm/types/llms/recraft.py new file mode 100644 index 0000000000..176810970b --- /dev/null +++ b/litellm/types/llms/recraft.py @@ -0,0 +1,17 @@ +from typing import Dict, List, Optional + +from typing_extensions import TypedDict + + +class RecraftImageGenerationRequestParams(TypedDict, total=False): + prompt: str + text_layout: Optional[List[Dict]] + n: Optional[int] + style_id: Optional[str] + style: Optional[str] + substyle: Optional[str] + model: Optional[str] + response_format: Optional[str] + size: Optional[str] + negative_prompt: Optional[str] + controls: Optional[Dict] \ No newline at end of file diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 45714a18d0..a025f387a1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2315,6 +2315,7 @@ class LlmProviders(str, Enum): LLAMA = "meta_llama" NSCALE = "nscale" PG_VECTOR = "pg_vector" + RECRAFT = "recraft" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 1556351e60..70fc77e467 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7151,6 +7151,12 @@ class ProviderConfigManager: ) return get_xinference_image_generation_config(model) + elif LlmProviders.RECRAFT == provider: + from litellm.llms.recraft.image_generation import ( + get_recraft_image_generation_config, + ) + + return get_recraft_image_generation_config(model) return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 92b48fba76..78fd56effc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16890,6 +16890,24 @@ "mode": "chat", "source": "https://platform.moonshot.ai/docs/pricing" }, + "recraft/recraftv3": { + "mode": "image_generation", + "input_cost_per_image": 0.04, + "litellm_provider": "recraft", + "supported_endpoints": [ + "/v1/images/generations" + ], + "source": "https://www.recraft.ai/docs#pricing" + }, + "recraft/recraftv2": { + "mode": "image_generation", + "input_cost_per_image": 0.022, + "litellm_provider": "recraft", + "supported_endpoints": [ + "/v1/images/generations" + ], + "source": "https://www.recraft.ai/docs#pricing" + }, "morph/morph-v3-fast": { "max_tokens": 16000, "max_input_tokens": 16000, diff --git a/tests/image_gen_tests/test_image_generation.py b/tests/image_gen_tests/test_image_generation.py index cc277f7481..79f7f42e55 100644 --- a/tests/image_gen_tests/test_image_generation.py +++ b/tests/image_gen_tests/test_image_generation.py @@ -165,6 +165,11 @@ class TestOpenAIGPTImage1(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: return {"model": "gpt-image-1"} +class TestRecraftImageGeneration(BaseImageGenTest): + def get_base_image_generation_call_args(self) -> dict: + return {"model": "recraft/recraftv3"} + + class TestAzureOpenAIDalle3(BaseImageGenTest): def get_base_image_generation_call_args(self) -> dict: litellm.set_verbose = True diff --git a/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py new file mode 100644 index 0000000000..4dd610ac86 --- /dev/null +++ b/tests/test_litellm/llms/recraft/image_generation/test_recraft_image_gen_transformation.py @@ -0,0 +1,270 @@ +import json +import os +import sys +from typing import List, Optional +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.recraft.image_generation.transformation import ( + RecraftImageGenerationConfig, +) +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + + +class TestRecraftImageGenerationTransformation: + def setup_method(self): + """Set up test fixtures before each test method.""" + self.config = RecraftImageGenerationConfig() + self.model = "recraft-v3" + self.logging_obj = MagicMock() + + + def test_map_openai_params_supported_params(self): + """Test that map_openai_params correctly maps supported parameters.""" + non_default_params = { + "n": 2, + "response_format": "url", + "size": "1024x1024", + "style": "photographic" + } + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False + ) + + assert result == non_default_params + + def test_map_openai_params_unsupported_param_drop_true(self): + """Test that map_openai_params drops unsupported parameters when drop_params=True.""" + non_default_params = { + "n": 2, + "unsupported_param": "value" + } + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=True + ) + + assert result == {"n": 2} + assert "unsupported_param" not in result + + def test_map_openai_params_unsupported_param_drop_false(self): + """Test that map_openai_params raises ValueError for unsupported parameters when drop_params=False.""" + non_default_params = { + "n": 2, + "unsupported_param": "value" + } + optional_params = {} + + with pytest.raises(ValueError) as exc_info: + self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False + ) + + assert "unsupported_param" in str(exc_info.value) + assert "is not supported for model" in str(exc_info.value) + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_get_complete_url_with_api_base(self, mock_get_secret): + """Test that get_complete_url returns correct URL when api_base is provided.""" + api_base = "https://custom.api.recraft.ai" + + result = self.config.get_complete_url( + api_base=api_base, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={} + ) + + expected_url = f"{api_base}/{self.config.IMAGE_GENERATION_ENDPOINT}" + assert result == expected_url + mock_get_secret.assert_not_called() + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_get_complete_url_with_secret_base(self, mock_get_secret): + """Test that get_complete_url uses secret when api_base is None.""" + mock_get_secret.return_value = "https://secret.api.recraft.ai" + + result = self.config.get_complete_url( + api_base=None, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={} + ) + + expected_url = f"https://secret.api.recraft.ai/{self.config.IMAGE_GENERATION_ENDPOINT}" + assert result == expected_url + mock_get_secret.assert_called_once_with("RECRAFT_API_BASE") + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_get_complete_url_with_default_base(self, mock_get_secret): + """Test that get_complete_url uses default base URL when no other options are available.""" + mock_get_secret.return_value = None + + result = self.config.get_complete_url( + api_base=None, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={} + ) + + expected_url = f"{self.config.DEFAULT_BASE_URL}/{self.config.IMAGE_GENERATION_ENDPOINT}" + assert result == expected_url + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_validate_environment_with_api_key(self, mock_get_secret): + """Test that validate_environment correctly sets authorization header when api_key is provided.""" + headers = {} + api_key = "test_api_key" + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=api_key + ) + + assert result["Authorization"] == f"Bearer {api_key}" + mock_get_secret.assert_not_called() + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_validate_environment_with_secret_key(self, mock_get_secret): + """Test that validate_environment uses secret API key when api_key is None.""" + mock_get_secret.return_value = "secret_api_key" + headers = {} + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None + ) + + assert result["Authorization"] == "Bearer secret_api_key" + mock_get_secret.assert_called_once_with("RECRAFT_API_KEY") + + @patch("litellm.llms.recraft.image_generation.transformation.get_secret_str") + def test_validate_environment_no_api_key_raises_error(self, mock_get_secret): + """Test that validate_environment raises ValueError when no API key is available.""" + mock_get_secret.return_value = None + headers = {} + + with pytest.raises(ValueError) as exc_info: + self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None + ) + + assert "RECRAFT_API_KEY is not set" in str(exc_info.value) + + def test_transform_image_generation_request(self): + """Test that transform_image_generation_request correctly transforms request parameters.""" + prompt = "A beautiful sunset over mountains" + optional_params = { + "n": 2, + "size": "1024x1024", + "style": "photographic" + } + litellm_params = {} + headers = {} + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers + ) + + assert result["prompt"] == prompt + assert result["model"] == self.model + assert result["n"] == 2 + assert result["size"] == "1024x1024" + assert result["style"] == "photographic" + + def test_transform_image_generation_response_success(self): + """Test that transform_image_generation_response correctly transforms successful response.""" + # Mock response data + response_data = { + "data": [ + {"url": "https://example.com/image1.jpg", "b64_json": None}, + {"url": None, "b64_json": "base64encodeddata"} + ] + } + + # Create mock response + mock_response = MagicMock() + mock_response.json.return_value = response_data + + # Create empty model response + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None + ) + + assert len(result.data) == 2 + assert result.data[0].url == "https://example.com/image1.jpg" + assert result.data[0].b64_json is None + assert result.data[1].url is None + assert result.data[1].b64_json == "base64encodeddata" + + def test_transform_image_generation_response_json_error(self): + """Test that transform_image_generation_response raises error when response JSON is invalid.""" + # Create mock response that raises JSON decode error + mock_response = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("Invalid JSON", "", 0) + mock_response.status_code = 500 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + with pytest.raises(Exception) as exc_info: + self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None + ) + + assert "Error transforming image generation response" in str(exc_info.value) \ No newline at end of file From 9022d144a6845286b81d801557a2482fc2b37005 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 21 Jul 2025 15:01:43 -0700 Subject: [PATCH 5/7] [Bug Fix] - gemini leaking FD for sync calls with litellm.completion (#12824) * bug fix - gemini leaking FD for sync calls * fixes for leaking FD --- litellm/llms/anthropic/chat/handler.py | 5 ++++- .../vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 04cf66cf02..620d2f0511 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + _get_httpx_client, get_async_httpx_client, ) from litellm.types.llms.anthropic import ( @@ -433,7 +434,9 @@ class AnthropicChatCompletion(BaseLLM): else: if client is None or not isinstance(client, HTTPHandler): - client = HTTPHandler(timeout=timeout) # type: ignore + client = _get_httpx_client( + params={"timeout": timeout} + ) else: client = client diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index d09599e878..e4a68dd82e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -35,6 +35,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + _get_httpx_client, get_async_httpx_client, ) from litellm.types.llms.anthropic import AnthropicThinkingParam @@ -1881,7 +1882,7 @@ class VertexLLM(VertexBase): if isinstance(timeout, float) or isinstance(timeout, int): timeout = httpx.Timeout(timeout) _params["timeout"] = timeout - client = HTTPHandler(**_params) # type: ignore + client = _get_httpx_client(params=_params) else: client = client From 1b05ea79cea4f4e61e5d9a7eeed8a250e19ae2d5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 21 Jul 2025 15:52:54 -0700 Subject: [PATCH 6/7] update docs --- docs/my-website/release_notes/v1.74.7/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/release_notes/v1.74.7/index.md b/docs/my-website/release_notes/v1.74.7/index.md index 26a09542ae..41e88824e0 100644 --- a/docs/my-website/release_notes/v1.74.7/index.md +++ b/docs/my-website/release_notes/v1.74.7/index.md @@ -28,14 +28,14 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.7 +ghcr.io/berriai/litellm:v1.74.7.rc.1 ``` ``` showLineNumbers title="pip install litellm" -pip install litellm==1.74.7 +pip install litellm==1.74.7rc1 ``` From 27c9be67ba5c303ee3ffb7c3a781363da0f45acc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 21 Jul 2025 16:28:02 -0700 Subject: [PATCH 7/7] [Feat] Add fireworks - `fireworks/models/kimi-k2-instruct` (#12837) * add fireworks - fireworks/models/kimi-k2-instruct * update source --- litellm/model_prices_and_context_window_backup.json | 13 +++++++++++++ model_prices_and_context_window.json | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 78fd56effc..cf7e3cb729 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14462,6 +14462,19 @@ "supports_tool_choice": false, "supports_response_schema": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct" + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "max_tokens": 16384, "max_input_tokens": 128000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 78fd56effc..cf7e3cb729 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14462,6 +14462,19 @@ "supports_tool_choice": false, "supports_response_schema": true }, + "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "input_cost_per_token": 0.6e-06, + "output_cost_per_token": 2.5e-06, + "litellm_provider": "fireworks_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct" + }, "fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": { "max_tokens": 16384, "max_input_tokens": 128000,