From 3ae92fbf30a6e98f1f38b27a7e675fc825658068 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 1 Apr 2026 19:06:33 -0700 Subject: [PATCH 01/29] fix: replace hardcoded url From f46074664e7ae8bfc142c33d266704b92e59ccef Mon Sep 17 00:00:00 2001 From: Vedanshu Joshi <40860706+Vedanshu7@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:42:35 -0400 Subject: [PATCH 02/29] fix(llm translation): redact Gemini API key from URL query params in error traces (#24943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy): use actual request start_time for failed spend logs async_post_call_failure_hook was calling datetime.now() for both start_time and end_time, making every failed request show Duration: 0.000s. litellm_logging_obj (already fetched in the same method for trace ID propagation) carries the real request start_time — use it as actual_start_time with a datetime.now() fallback when absent. Add two regression tests covering the fix and the fallback path. Fixes #24888 * fix(llm translation): redact Gemini API key from URL query params in error traces Gemini API requests authenticate via a ?key= URL query param. When a provider call fails, httpx.Response.raise_for_status() embeds the full URL in the error message, leaking the key in exception traces and logs. Changes: - Extract secret-redaction logic from litellm/_logging.py into a new public utility module litellm/litellm_core_utils/secret_redaction.py, exposing redact_string() as a proper public API instead of a private helper - Add (?<=[?&])key=[^\s&'"]{8,} pattern to _SECRET_RE so ?key=VALUE and &key=VALUE fragments are caught by the existing SecretRedactionFilter - Apply redact_string() to error_str in exception_mapping_utils.py so the key is also stripped from the mapped exception message surfaced to callers - Add 5 regression tests covering: ?key=, &key=, short-value no-op, httpx raise_for_status path, and end-to-end logger output - Keep _redact_string = redact_string alias in _logging.py for backward compat Fixes #24902 * revert: undo start_time fix for failed spend logs * fix: gate exception redaction on _ENABLE_SECRET_REDACTION opt-out flag - Apply redact_string() conditionally in exception_mapping_utils.py, matching the same _ENABLE_SECRET_REDACTION guard used by SecretRedactionFilter so that LITELLM_DISABLE_REDACT_SECRETS=true is honoured for exception messages - Rewrite test_redact_string_applied_to_httpx_error_message to use pytest.raises so assertions cannot be silently skipped if raise_for_status() doesn't raise - Add test_exception_mapping_respects_redaction_opt_out to verify the flag is respected end-to-end through exception_type() --- litellm/_logging.py | 5 +- .../exception_mapping_utils.py | 15 ++++- .../litellm_core_utils/secret_redaction.py | 66 +++++++++++++++++++ tests/test_litellm/test_secret_redaction.py | 13 ++-- 4 files changed, 86 insertions(+), 13 deletions(-) create mode 100644 litellm/litellm_core_utils/secret_redaction.py diff --git a/litellm/_logging.py b/litellm/_logging.py index d072cc549d..73a0846983 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,14 +1,14 @@ import ast import logging import os -import re import sys from datetime import datetime from logging import Formatter -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads +from litellm.litellm_core_utils.secret_redaction import redact_string set_verbose = False @@ -21,7 +21,6 @@ _ENABLE_SECRET_REDACTION = ( os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true" ) -_REDACTED = "REDACTED" def _build_secret_patterns() -> re.Pattern: diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 5a7d4e33b6..cc665231d1 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -6,7 +6,8 @@ from typing import Any, Optional import httpx import litellm -from litellm._logging import _redact_string, verbose_logger +from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger +from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -261,10 +262,18 @@ def exception_type( # type: ignore # noqa: PLR0915 original_exception=original_exception ) try: - error_str = str(original_exception) + error_str = ( + redact_string(str(original_exception)) + if _ENABLE_SECRET_REDACTION + else str(original_exception) + ) if model: if hasattr(original_exception, "message"): - error_str = str(original_exception.message) + error_str = ( + redact_string(str(original_exception.message)) + if _ENABLE_SECRET_REDACTION + else str(original_exception.message) + ) if isinstance(original_exception, BaseException): exception_type = type(original_exception).__name__ else: diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py new file mode 100644 index 0000000000..4473c6539c --- /dev/null +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -0,0 +1,66 @@ +""" +Credential/secret redaction utilities. + +This module owns the compiled regex and the public `redact_string` helper so +that any part of the codebase (logging, exception mapping, etc.) can scrub +secrets from strings without depending on the logging-configuration module. +""" + +import re +from typing import List + +_REDACTED = "REDACTED" + + +def _build_secret_patterns() -> "re.Pattern[str]": + patterns: List[str] = [ + # AWS access key IDs + r"(?:AKIA|ASIA)[0-9A-Z]{16}", + # AWS secrets / session tokens / access key IDs (key=value) + r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)" + r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}", + # Bearer tokens (OAuth, JWT, etc.) + r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", + # Basic auth headers + r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", + # OpenAI / Anthropic sk- prefixed keys + r"sk-[A-Za-z0-9\-_]{20,}", + # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) + r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", + # x-api-key / api-key header values (handles 'key': 'value' dict repr) + r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", + # Anthropic internal header keys + r"x-ak-[A-Za-z0-9\-_]{20,}", + # Google API keys (bare key value) + r"AIza[0-9A-Za-z\-_]{35}", + # URL query-param key=VALUE (e.g. ?key=AIza... or &key=...) — catches the + # full "key=" fragment so the value is redacted regardless of format. + r"(?<=[?&])key=[^\s&'\"]{8,}", + # Password / secret params (handles key=value and 'key': 'value') + r"\w*(?:password|passwd|client_secret|secret_key|_secret)" + r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", + # Database connection string credentials (scheme://user:pass@host) + r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", + # Databricks personal access tokens + r"dapi[0-9a-f]{32}", + # ── Key-name-based redaction ── + # Catches secrets inside dicts/config dumps by matching on the KEY name + # regardless of what the value looks like. + # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." + r"(?:master_key|database_url|db_url|connection_string|" + r"private_key|signing_key|encryption_key|" + r"auth_token|access_token|refresh_token|" + r"slack_webhook_url|webhook_url|" + r"database_connection_string|" + r"huggingface_token|jwt_secret)" + r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", + ] + return re.compile("|".join(patterns), re.IGNORECASE) + + +_SECRET_RE = _build_secret_patterns() + + +def redact_string(value: str) -> str: + """Scrub known secret/credential patterns from *value* and return the result.""" + return _SECRET_RE.sub(_REDACTED, value) diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 6cbb2fd7b8..5c9b3b37df 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -7,13 +7,12 @@ import pytest from litellm._logging import ( JsonFormatter, - _redact_string, _secret_filter, - _setup_json_exception_handlers, verbose_logger, verbose_proxy_logger, verbose_router_logger, ) +from litellm.litellm_core_utils.secret_redaction import redact_string SECRET = "sk-proj-abc123def456ghi789jklmnopqrst" @@ -57,12 +56,12 @@ def test_redact_string_catches_secret_patterns(): SECRET, ] for secret in cases: - result = _redact_string("msg: " + secret) + result = redact_string("msg: " + secret) assert secret not in result, f"{secret!r} was not redacted" assert "REDACTED" in result normal = "Loaded model gpt-4 with 3 replicas on us-east-1" - assert _redact_string(normal) == normal + assert redact_string(normal) == normal def test_filter_redacts_secrets_in_logger_output(): @@ -155,7 +154,7 @@ def test_x_api_key_regex_does_not_consume_json_delimiters(): """x-api-key pattern must stop before closing quotes/braces so JSON stays valid.""" # Simulates a JSON log line containing an x-api-key header value json_line = '{"headers": {"x-api-key": "secret123"}, "status": 200}' - result = _redact_string(json_line) + result = redact_string(json_line) # The secret value should be redacted assert "secret123" not in result assert "REDACTED" in result @@ -234,12 +233,12 @@ def test_key_name_redaction_catches_secrets_in_dict_repr(): "'slack_webhook_url': 'https://hooks.slack.com/services/T00/B00/xxx'", ] for secret_line in cases: - result = _redact_string(secret_line) + result = redact_string(secret_line) assert "REDACTED" in result, f"Key-name redaction missed: {secret_line!r}" # Non-sensitive keys should NOT be redacted safe = "'enable_jwt_auth': True, 'store_model_in_db': True" - assert _redact_string(safe) == safe + assert redact_string(safe) == safe def test_key_name_redaction_in_general_settings_dict(): From 49ec6aba80d33b103a9d3d9e12bce57881510ae1 Mon Sep 17 00:00:00 2001 From: Mathieu St-Vincent <6082025+mats852@users.noreply.github.com> Date: Wed, 1 Apr 2026 23:05:31 -0400 Subject: [PATCH 03/29] feat: add Qohash Nexus guardrail hook (#24927) * feat: added Qohash Nexus guardrail hook * fix: ui_friendly_name of Qostodian Nexus * Update litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * Update litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../guardrail_hooks/qohash/__init__.py | 35 +++ .../guardrail_hooks/qohash/qohash.py | 74 +++++ litellm/types/guardrails.py | 5 + .../guardrails/guardrail_hooks/qohash.py | 16 ++ .../test_qostodian_nexus_guardrail.py | 252 ++++++++++++++++++ .../public/assets/logos/qohash.jpg | Bin 0 -> 11581 bytes .../guardrails/guardrail_info_helpers.tsx | 2 + 7 files changed, 384 insertions(+) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/qohash/__init__.py create mode 100644 litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/qohash.py create mode 100644 tests/test_litellm/proxy/guardrails/test_qostodian_nexus_guardrail.py create mode 100644 ui/litellm-dashboard/public/assets/logos/qohash.jpg diff --git a/litellm/proxy/guardrails/guardrail_hooks/qohash/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/qohash/__init__.py new file mode 100644 index 0000000000..465f52db3d --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/qohash/__init__.py @@ -0,0 +1,35 @@ +from typing import TYPE_CHECKING + +from litellm.types.guardrails import SupportedGuardrailIntegrations + +from .qohash import QostodianNexus + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): + import litellm + + _instance = QostodianNexus( + api_base=litellm_params.api_base, + guardrail_name=guardrail.get("guardrail_name", ""), + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + additional_provider_specific_params=litellm_params.additional_provider_specific_params, + extra_headers=getattr(litellm_params, "extra_headers", None), + ) + + litellm.logging_callback_manager.add_litellm_callback(_instance) + + return _instance + + +guardrail_initializer_registry = { + SupportedGuardrailIntegrations.QOSTODIAN_NEXUS.value: initialize_guardrail, +} + + +guardrail_class_registry = { + SupportedGuardrailIntegrations.QOSTODIAN_NEXUS.value: QostodianNexus, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py new file mode 100644 index 0000000000..7c856b676b --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py @@ -0,0 +1,74 @@ +""" +Qostodian Nexus (by Qohash) — LiteLLM guardrail integration. +""" +import os +from typing import TYPE_CHECKING, Literal, Optional, Type + +from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + GenericGuardrailAPI, +) +from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, +) +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +GUARDRAIL_NAME = "qostodian_nexus" + + +class QostodianNexus(GenericGuardrailAPI): + def __init__( + self, + api_base: Optional[str] = None, + **kwargs, + ): + api_base = api_base or os.environ.get("QOSTODIAN_NEXUS_API_BASE", "http://nexus:8800") + + kwargs["guardrail_name"] = kwargs.get("guardrail_name", GUARDRAIL_NAME) + + # Merge built-in Qostodian Nexus identifier headers with any caller-supplied extra_headers + nexus_headers = [ + "x-qostodian-nexus-identifiers-trace", + "x-qostodian-nexus-identifiers-source", + "x-qostodian-nexus-identifiers-container", + "x-qostodian-nexus-identifiers-identity", + ] + + existing = kwargs.get("extra_headers") or [] + kwargs["extra_headers"] = nexus_headers + [h for h in existing if h not in nexus_headers] + + super().__init__( + api_base=api_base, + **kwargs, + ) + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> GenericGuardrailAPIInputs: + """ + Apply Qostodian Nexus to the given inputs. + + NOTE: This override is intentionally a pass-through. It must be present + directly in this class's __dict__ so that LiteLLM's unified guardrail + routing check (`"apply_guardrail" in type(callback).__dict__` in + litellm/proxy/utils.py) routes calls correctly. Do not remove. + """ + return await super().apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type=input_type, + logging_obj=logging_obj, + ) + + @classmethod + def get_config_model(cls) -> Optional[Type[QostodianNexusConfigModel]]: + """ + Returns the config model for Qostodian Nexus. + """ + return QostodianNexusConfigModel diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index a98f9d666a..04347aebe3 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -38,6 +38,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( from litellm.types.proxy.guardrails.guardrail_hooks.hiddenlayer import ( HiddenlayerGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, +) """ Pydantic object defining how to set guardrails on litellm proxy @@ -96,6 +99,7 @@ class SupportedGuardrailIntegrations(Enum): AKTO = "akto" MCP_JWT_SIGNER = "mcp_jwt_signer" LLM_AS_A_JUDGE = "llm_as_a_judge" + QOSTODIAN_NEXUS = "qostodian_nexus" class Role(Enum): @@ -773,6 +777,7 @@ class LitellmParams( QualifireGuardrailConfigModel, BlockCodeExecutionGuardrailConfigModel, HiddenlayerGuardrailConfigModel, + QostodianNexusConfigModel, ): guardrail: str = Field(description="The type of guardrail integration to use") mode: Union[str, List[str], Mode] = Field( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/qohash.py b/litellm/types/proxy/guardrails/guardrail_hooks/qohash.py new file mode 100644 index 0000000000..5abfa69148 --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/qohash.py @@ -0,0 +1,16 @@ +from typing import Optional + +from pydantic import Field + +from .base import GuardrailConfigModel + + +class QostodianNexusConfigModel(GuardrailConfigModel): + api_base: Optional[str] = Field( + default=None, + description="The API base URL for Qostodian Nexus. If not provided, the `QOSTODIAN_NEXUS_API_BASE` environment variable is checked. Defaults to http://nexus:8800.", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "Qostodian Nexus" diff --git a/tests/test_litellm/proxy/guardrails/test_qostodian_nexus_guardrail.py b/tests/test_litellm/proxy/guardrails/test_qostodian_nexus_guardrail.py new file mode 100644 index 0000000000..6daa3e1430 --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/test_qostodian_nexus_guardrail.py @@ -0,0 +1,252 @@ +""" +Unit tests for Qostodian Nexus (by Qohash) integration. + +Tests verify: +1. QostodianNexus can be instantiated with default and custom values +2. Qostodian Nexus is registered in SupportedGuardrailIntegrations +3. Guardrail initializer and class registries contain Qostodian Nexus +4. Configuration parameters are properly passed through +5. QostodianNexusConfigModel works correctly +""" + +import os +import pytest +from unittest.mock import MagicMock + + +def test_qostodian_nexus_initialization_with_defaults(): + """Test QostodianNexus initializes with default values.""" + import os + from unittest.mock import patch + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + # Unset env var so the hardcoded default is used + env = {k: v for k, v in os.environ.items() if k != "QOSTODIAN_NEXUS_API_BASE"} + with patch.dict(os.environ, env, clear=True): + guardrail = QostodianNexus() + + # Should use default api_base + assert guardrail.api_base is not None + assert "nexus:8800" in guardrail.api_base + + +def test_qostodian_nexus_initialization_with_custom_api_base(): + """Test QostodianNexus initializes with custom api_base.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + custom_api_base = "http://custom-nexus:9000" + guardrail = QostodianNexus(api_base=custom_api_base) + + assert custom_api_base in guardrail.api_base + + +def test_qostodian_nexus_in_supported_guardrail_integrations(): + """Test that Qostodian Nexus is registered in SupportedGuardrailIntegrations enum.""" + from litellm.types.guardrails import SupportedGuardrailIntegrations + + # Check enum contains QOSTODIAN_NEXUS + assert hasattr(SupportedGuardrailIntegrations, "QOSTODIAN_NEXUS") + assert SupportedGuardrailIntegrations.QOSTODIAN_NEXUS.value == "qostodian_nexus" + + # Check it's in the list of all values + all_values = [e.value for e in SupportedGuardrailIntegrations] + assert "qostodian_nexus" in all_values + + +def test_qostodian_nexus_in_guardrail_initializer_registry(): + """Test that Qostodian Nexus is registered in guardrail_initializer_registry.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import ( + guardrail_initializer_registry, + ) + + assert "qostodian_nexus" in guardrail_initializer_registry + assert callable(guardrail_initializer_registry["qostodian_nexus"]) + + +def test_qostodian_nexus_in_guardrail_class_registry(): + """Test that Qostodian Nexus is registered in guardrail_class_registry.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import ( + guardrail_class_registry, + QostodianNexus, + ) + + assert "qostodian_nexus" in guardrail_class_registry + assert guardrail_class_registry["qostodian_nexus"] == QostodianNexus + + +def test_qostodian_nexus_config_model_initialization(): + """Test QostodianNexusConfigModel can be instantiated.""" + from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, + ) + + config = QostodianNexusConfigModel( + api_base="http://test:8800", + ) + + assert config.api_base == "http://test:8800" + + +def test_qostodian_nexus_config_model_defaults(): + """Test QostodianNexusConfigModel uses correct defaults.""" + from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, + ) + + config = QostodianNexusConfigModel() + + assert config.api_base is None + + +def test_qostodian_nexus_config_model_ui_friendly_name(): + """Test QostodianNexusConfigModel returns correct UI friendly name.""" + from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, + ) + + ui_name = QostodianNexusConfigModel.ui_friendly_name() + assert ui_name == "Qostodian Nexus" + + +def test_qostodian_nexus_initializer_function(): + """Test the initialize_guardrail function.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import initialize_guardrail + from litellm.types.guardrails import LitellmParams, Guardrail + from unittest.mock import patch + + # Mock litellm.logging_callback_manager + with patch("litellm.logging_callback_manager") as mock_manager: + mock_manager.add_litellm_callback = MagicMock() + + # Create test params + litellm_params = LitellmParams( + guardrail="qostodian_nexus", + mode="pre_call", + api_base="http://test:8800", + default_on=True, + ) + + guardrail_config: Guardrail = {"guardrail_name": "test-qostodian-nexus"} + + # Call initializer + result = initialize_guardrail(litellm_params, guardrail_config) + + # Verify callback was added + mock_manager.add_litellm_callback.assert_called_once() + + # Verify returned instance has correct properties + assert result is not None + assert "test:8800" in result.api_base + + +def test_qostodian_nexus_inherits_from_generic_guardrail_api(): + """Test that QostodianNexus inherits from GenericGuardrailAPI.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( + GenericGuardrailAPI, + ) + + assert issubclass(QostodianNexus, GenericGuardrailAPI) + + +def test_qostodian_nexus_guardrail_name_constant(): + """Test that GUARDRAIL_NAME constant is defined correctly.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash.qohash import GUARDRAIL_NAME + + assert GUARDRAIL_NAME == "qostodian_nexus" + + +def test_qostodian_nexus_get_config_model(): + """Test that QostodianNexus returns the correct config model.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, + ) + + config_model = QostodianNexus.get_config_model() + + assert config_model is not None + assert config_model == QostodianNexusConfigModel + + +def test_qostodian_nexus_env_vars(): + """Test that QOSTODIAN_NEXUS_API_BASE env var is picked up correctly.""" + import os + from unittest.mock import patch + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + with patch.dict(os.environ, {"QOSTODIAN_NEXUS_API_BASE": "http://new-api:8800"}): + guardrail = QostodianNexus() + assert "new-api:8800" in guardrail.api_base + + +def test_qostodian_nexus_config_model_field_descriptions(): + """Test that QostodianNexusConfigModel has correct field descriptions.""" + from litellm.types.proxy.guardrails.guardrail_hooks.qohash import ( + QostodianNexusConfigModel, + ) + + # Check that field descriptions mention the correct env vars + api_base_field = QostodianNexusConfigModel.model_fields["api_base"] + assert "QOSTODIAN_NEXUS_API_BASE" in api_base_field.description + + +def test_qostodian_nexus_unified_detection(): + """ + Test that QostodianNexus is properly detected by LiteLLM's unified guardrail system. + + This verifies the fix for the detection bug where QostodianNexus wasn't being + recognized because apply_guardrail was only inherited, not in the class's own __dict__. + """ + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + # Create an instance (this is how LiteLLM uses it) + instance = QostodianNexus(api_base="http://test:8800") + + # Test the exact detection logic used in litellm/proxy/utils.py:868 + # use_unified = "apply_guardrail" in type(callback).__dict__ + use_unified = "apply_guardrail" in type(instance).__dict__ + + # Should be detected as using unified guardrail system + assert use_unified is True, ( + "QostodianNexus should be detected by unified guardrail system. " + "The apply_guardrail method must be present in QostodianNexus.__dict__" + ) + + # Also verify the method is callable + assert hasattr(instance, "apply_guardrail") + assert callable(instance.apply_guardrail) + + +def test_qostodian_nexus_builtin_extra_headers(): + """Test that QostodianNexus includes built-in x-qostodian-nexus-identifiers-* headers.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + instance = QostodianNexus() + + expected_headers = [ + "x-qostodian-nexus-identifiers-trace", + "x-qostodian-nexus-identifiers-source", + "x-qostodian-nexus-identifiers-container", + "x-qostodian-nexus-identifiers-identity", + ] + + for header in expected_headers: + assert header in instance.extra_headers, ( + f"Expected built-in header '{header}' to be in extra_headers" + ) + + +def test_qostodian_nexus_extra_headers_merged(): + """Test that caller-supplied extra_headers are merged with built-in headers.""" + from litellm.proxy.guardrails.guardrail_hooks.qohash import QostodianNexus + + custom_header = "x-custom-correlation-id" + instance = QostodianNexus(extra_headers=[custom_header]) + + # Built-in headers should be present + assert "x-qostodian-nexus-identifiers-trace" in instance.extra_headers + # Custom header should also be present + assert custom_header in instance.extra_headers + # No duplicates + assert len(instance.extra_headers) == len(set(instance.extra_headers)) diff --git a/ui/litellm-dashboard/public/assets/logos/qohash.jpg b/ui/litellm-dashboard/public/assets/logos/qohash.jpg new file mode 100644 index 0000000000000000000000000000000000000000..50227ab3910ca8874f2e9ca744fbe03af1462a3b GIT binary patch literal 11581 zcmdtIc|4Tw|1W;qB4V;{$-ZUFUQEWmhU{CYWDlt@GK?i_mMDZ!*^(`?WXWV1LWu0L z%qSwuOflUt#<}Z#KA&^G=kfiX$9bId`2F)6<1udcHP?N;uIu%BzMik=bsfzdEdwV_ z4NVLIDk>_#82SN@<^jDwSN~71Kb`zPb@1oQNACbu2H-vLo`#AQpk}3_VWm3i1q1{eIl zMS?k$VltmGi0amNaaxZN#FVeyiapNA#m&RZCoUl=C4ELkRZacec?~^%14AQY6H^;o zJ9`I5CufiAo?hNQzJ9mwgoK76!|%r3i%&>QN>0go_$WIk_iIH7>BaW-zaJPJ|1>c>tDa7rUTIprWRwrJ<$&V;2>581zfSN=qlKc#KWQivDUayNFT@1BY(rv-+;%qRQ3; z&TF^E7`en$aN@*2rv0_-|ID!1|0~P>ZP@>?YYsR;Lj^4!4J&{EC=@B+DRnd>@c;b> z#r+6a?!XY-1$mBuE(Lfb0kd4Z3H`4)VTm7~mE zviR+Dj*@cfZl8K>AiJNO9B?ImM6&XANdanx-F1hck@5~~O09rH$=J(f+R7U{o z19X=T`r8ub`Vl~8C0ic>Yq{F=WJ8@JfR646D2Y0JcmxoPh-ODXkt=|9fG%`*KpE^b znbUt6k-B#TB)JI?5@RSD-y(HFacr?2A8i{~NghfQ&n!Rh+%XX%gOtZJ2ZfxP zi^rFJv8eyx5KVPZtW5`S0GFu#%MYrxBY=}40h%q3_PC!y*%KGbj%DIti4>kGxC5S9 z7q6ct^r_Kby3U(%T&Xh0iu)lWoBEK5!d>79;8OslDz?(_#q<57ci_;0DKaPGb-;=x z+d|j!md3V;QzL@f*JjukXD%+1WkT2Tu>9VeH!Z12B^|A;3}BIgbATA1; z_os;@hC>XT9*s|Pi$pB%^!?sF0^$)Iw(#=tDQ34_BV1urL`X>T5m3K3l%zK#k{J{b z9(>-@`0+>j{fRkO7kB4*x7kt7;j%9$7o0zLe-~4pALSc5F=kv^)h`=Zn_ik=PkV|F zsQ)i#$jyU??i-Np64;+m!#!MFp14e3hoxt!LjaM)GCP>=e}fl8gb z(T%U>!PwthyPKSb`7|Dakb(3p3YSLo247=w=}C=)I-XgCXOdpV^lO~|8V*`tkPuKu=HRwHOhX4* z(v#JQT#Yw6nJD97q$L>HGay(jc*b{8qSern0Zh?-?+91l$alGM=S0i5%bXXZVSDyQ zPMoeh8X8q4(sWLa)YYXo^{5#{V_6IUjsFckz*Yq*EccFp*hHtOL3F+AugxtSbCj#* z3T(^71C1+S*c6yLumB^1R6Dd3Cq592vu1c}w%IM4-n^GnedqY&b~NnA5%7KLe>?(H zwvnHnT)DnKk{NNk!mrJIuePk*kLy+kcc?l^xoo&>p>ga1(K0BQ*pz~I5)KQ8zcrnH zk4cF9@UXhiz-t(YYWlCE)~@qy^QY?CJ*L^L#gXnQ{Ev-r9@(4o&b>cJW$JO(=Q0bE zmjQDC{a}>LZAAK#OWKC!BLU@3+|Il^jbM&Cy`hYf1>Jcb4KN`ix?jI7qr8YV5z}l+ z3s9)dF%GS6zVGG8Wuq;azWc|G1&XQ&Kl|Dw<4}r3dON~mmB%uM*L8z*nwKbmEv;X7 z#%{;U-rSgK4!iMO+N&YUmff)0@fdUJZS7N@f1H7SPx=TrRvooyMHzN)Uf9`OTM@YE zj5>GtVw2sVLHVNfVh=tvwJ0cIZC30)W14fXo9}C5jt}E{o^0E+bkHdBYk9ZWc$PRHnwmfS@uJ3)}ObDSeD5!pawa@9wJuoINX8w z9sjE*BhHQfRqI5D15pyYFM02dijW*j&72Y?EVvNU<}~J5_e~YbO|U6)lo2k1}Om z6;OX1U)E;CuP7Fce*ATz2AeiHoc&qJWnuQSxqnE|nMXNH_rv+5ahWWP0~|^3H^S|$ zVIt8&5TI&U(jzgw7#P_JS9c0jcj)*Ccfv8>zDSfz*lRxyuUe3C9;*|^drRtg1D9t zJ&rP9{(y8yFxd%M+~UmE>sNZ8XK1aBF#7hNCU0t)WL#5hL_so_lpHQA{Oj#|={#1Y zS%#9)FH%1A?gIJHB1;|t;JFG4BfIxOSkz*Xa1+6>EG(E1x6D~M-HoBw{?tC>r?H*= z(yih?*)rgymc6POxZIdvyU;mpnKnrpGytNG{f!v3M=`5n9|MAgW6vcT*M0yzlMd}lfHNc-Q>zvw^VvJ+E?6d#qq1c0}n^`hv?qPu_!GA`41wrtw zAaiJ51bGj%!BLVTm{I-N1vDeYzj-x5;o(nS+T?TSO*=D3*x=GR_4^Wxb^Yf2%d=9> z+wY}Vex?HGlDGdt-T#>Eqo8qhg&_i;9UO!HxS%D6{G~l{r3oxvInW1P34=4kY;FQ* zEUt0>t@E7Zl^^mkWo8jo7ca3@`UU?yDMdq0k}UGiD{p<1W3nwTo)x~9H)rCN@1SHR z+i+e%EK5_lyqQZ1 zdQP3O-Lq^waheoTh!^8=MxNbc(TLdy%JH&;)5tIq=i&WMhNzK!@oK9LH!eb9+OIT) zoU;!%C~8yJ%=LQJr_CFc2U~9Qt1cAQJ3r;1`V|kL;qHW6iJRO32iMSDhz^GpG_&uq z9sKC#niz(SEEnwW=l%9AVXIWOQ(;WEH%%p3?MYRbpr)`|!+Cl*fXk=-X|kg5HNS$%`}!O+=9it zO(V;tDix)H8^UigG`(7~9E+tk8R!q-ly^r!lTjif05QwG?7k(68@eCno~%XvaW67( zkoS3xD}QhI$T^f=i5o-SJ%_uYl>);Z#8aFhqGdp2%zr{p*Z{Ey0IwJU5HPtA4Y}St z#2E@PLGu}N{iGjLohLwr-f3K68PPM6zhNfn!1w%o& zUAc#}`H?>|_7b8I`E7qt74{uLVgBRDNN38JdlM`P)!Hy zNw(^EsK(mI__&-;E;sLmDbIrjh6!8>M7wN1Qfc$Hjbv8%?;BP<^CQ)}`)W)TMwV{m zI53=M8rHOG&KV(1T|W=w#5Rs&MKTkNg!Z8E^FdEsih8FGFE%(+I|B0yxF{S%1?>>^H)uIKjItg|5%aDk~}Ou^MC9ip{X?$h{-iE8;b zUN_fN4hRk(EA|XNJ8+4C9^3>U_G)16qYUbhb?zMnR?{f+8owf;sV1gGU&+qNrh@f7 z0mC1@JUQWkf!kjBhS$uLvrHtS6@7rG%*8Co2q$77<~iuz(-zrRRfkNptslD+pf;|S zvw8$rPtFsvUiJ3%pkxiT8JL|ZFc(o7sFDVNNk zNPh4zXnWZ2CfC%|IVoE!X4N{9!4r3p;lmI|&d=XSmRnT&X@6Xfirfl;#3#%v*`>x` zo0k3udPqtAf=giDf&9V(n7*#^-A1dVimWk-dJNa|`OcSDe&6V^(aXLR*rT)%>Fz(g z_u8F5gyHnofg&o3n6cS(piLG#d*K zOfbnN2U^d{-NQ)^Ih$d6aQQ@tb?HOi_#X=%!By-LK<`aiJWd=UwGfIoGnR`_@?)qe z!+bY#wN#qwAFCCkiDp>)oUknMo*q%75cNbA`sOC>cLipJ*86N+wl|*&4v7EdF}t%+ zgtbL=)g4$<7Md~q+c0#QE-rKaLj2)3oICsU@v8VZF{*3|F{-I)x zTpwmuglG=+pm6(B*h)>G>gHc_ZdJajX?xXU zZ#AqUqr=I6v*7{y}Z zrgDEuAqW|+s@WN-yUgdyB1f9}I;PsXoei1JY_!NHf9?IMjcC35zwj{7hQerfa0Q7b zaUG^mG$~{I6CVrpx{4Sl@Pxnu*-G~BMH-V&Bh|a_!o~NSR$t%tYv4TL5p5<-EBq8j zECDDjdI)tff9FuUf~ATymcvo8bklHw?2P7HsTVBdYQWKfh*uOA->>O0%P9 zqyvH>AER5)w0JwY$0Q(Q-kkIE(nRf6wQPWp?$65$3m+c1bg@avj+?0@B|T+xg1~a` z3j{U-ly`gzWcFVqHqd{0pjscn9wq6^r{24CUp=+dQZl(|K~1yWTwZ$_?xFjscIB0i ziHyMe*12OV4`4I@Hk)FCUj7~gO4?!icV-j%Nn-@h)N!)mVg8+&O_UYg6A7b93eQG< zbC5*Nx!e!=0qiz!Q|_Rj^!vl}sLv$>f~!4$<7`iI)Ahgxxk6GnERSNNj4c9hiY|EI zMpGIWEcOR^FX~+;thYxaXbiDG`w9JIwsVu#=O(RK0b6}-PCMHx zXI@5gKq40v7OMB)2l$}gB}SAup=>D?=t~_&-TR@k4^lvN)f5^{cj(1moC~HuYKU2)Mx3wnO+3dgx`Ol!N;R4kb1N2c?8@; zAATfPQ6xlrMt>gxd1##A5pYLqgg|L8M;7aklXXfim#yYjJGm^)e3mqk+ZJUL7JZ@w zNeSTPvs>;ho-<^5|CsL8b7x*Qv-9f~iNrf@oe2>sjpXtqYCbw-IpW3FMon#`-sbH&LBSce?ZgeOqurQ(%3Sf`osROv8C2A}{nO?zyM|VpE?SJB-CF)D7SLUwYiY3 zRKF8G)Y0hcyZ=Zfe?3$19A(&s&XdyxTEOCBC5GuHQEK@P;QR?76aR+skXx5ki zD=lQb$k*F(M|J3TE%phU#g4+P3V2$ENi0;$uB=h<#2Em9KZnpci^ua#z6{c;G(Vh; z$3rJK%gIs9=jkn{h1eGYCe+c0DGv(s^5H$pO#`S@B7&$z@lPK0U?uUZT&Emadm%7$ zVa}*(z8Ex6aVl3V9hXap6H|8)KLViK3BQ8M6B2BH1hGqe7@0$F#h7T6dV|8TuC!)O zxZb_vi2R}{uUhq3krdA9SjA-QWK3W3MvPhp3|bfL`>s0jf{?=&*8Sxn=}kYcrfuJ3 z1^K+#Z}<8*`^~RE5lLE^9RY`xPd?({Vmt8z-dpuq_X=Nn_i8j#k<;AUP@S=HSnTbq}I-lH!(j@lVY*D?xFH3cooFLV0 zBTCTxJ<0=T2BH}vx@Qc|V zVMVo)t6t;OY!Ky0?9w{>QA-%9*}bSqs3G&!C3JY;4kdf%SjqxKhJqqX{X@_CX+%ya z313I9HN8pKhYG;6LYdkE7yey!QIQ4ET}w-ZOU$O4OYsSyzQ^A;fLivF%e+h9t1eo_r%b0V^(LS4d3r`9{t`s`u0_MYS{_(v#Q|Vkv8wwFNn-2z%L5*+{D1V@}?wirgU9BGf~ECVlYwnev&V^ zeR42RL#F*Q@!&uoNs6PkjXR9*C*XnL!=X4Ua1E6n`1jIs8Bq#rfwF{B+$6FSGK z+#`AT^{GJb`vsNB{nz~P{wD1$8V$GZl1yIIiDmp)Q~nTqzS$$~1Mzl<;<-(O%jUjX zud5^_-m>}042ZJ4VPT(x;4OK&__&q~XoDsF;A~2u$RQjuq9zKAC&bd3CLg|#;bT=7 zS{Yh7*16QvDOJ;b%Jq@AO6iIzPtsLl=G#XEWU^slh}>ieSH;T%$$4Zrmh+9?{aa7) zPx8x7=${4OFAh4w^-C<;a?(_JIzvJ_?h1aAC`S9gbe!E#Qx+qIO?_N&G3P&KaXQ9!k&Wra0H# znyPm#q3Jq5f~qAH-DXC#d;)854{kM5QIw8=7KhCme8x-_sTLeNP(#-8H?@NSD*|R0 z$7^@wzBEU=^xVEUttBue;&fa|r_xtx?lV^Db?M;2WYYPkuq>!dfq-zlpTevSIcm%X zx-n=o10OVNgqKV*tRym#Y>Q1_5ehcc{q~zH2ywmUcTvhG%p@e+nh9xNv)2;i5^^{1 z-u3xt?BOKDLUrizFVaqbA1XlwI*uPGp;*9Y#GiPt=WX6G6i%e36EOjE%Sk`W#n(Q| zQXuZM)-2^CS7L??)~n<8KIPy-4g?i%Jo8m^3X6~>FPQ9DPrDP4vs4Z zyxw6~UZl{A=OqCwrBG=y@twk43>kcP15qEesT@FL!b`*im7_ckOY64akyV=ARh7QQ zurIHga=Z9Ad+)ujQn6X^n^r`{MSg8FIStaBgniqFVw@ZXb&`p?%)LJnLwIP?9wcdO zHw`y@@E+CD-fC#ZWYT@Oa^h-*&rD&hRnh<_`RZ=o3TIN^SF&JS_3yL39zUhJ-8QT2 z%&&eM8A2MIf784<*Jebl8+S?NM0vS)^O%l*=_3s(*nnzY&$Q8yM(&UJP)=HK}hq<1j-?Ps6D1>D^{ zF8Pj^tLHg>+f+(Qk5V~X&ZtI)`>L#OFBVqke&a;z6@PrDE!g8g7WfL~(ESOrPB}{X zurrDuyzXX#J3xRA7yN=&S7i=U9AYump z3K$cRa42fIwV+OBA0>exf;eQ~IUxdiMutT!yQlJbrj|L_5+qXs#Orgyp9_@^A^ zYl+1ee;zJtJo8QRwzlA=FmCju4@H(hNycZumUJq=xrvP5ISewKsJUzsfG+pTsdt-M z=n}%=-ZbZ*_XzHhq1O0#Z1^AW7sO)70y_s03NcZ~Q7VVAWY(%@(@`o*{Y3f|+R!lR zHQ!D;f<;x?5@@4~F)sh<5yS?TU7gfd)_+NgIwiK9{#A*J68@J2309*C29`5kDlOpyL?7bK6^g$TY4utNHBGRq9MX*Km2!F#Zvn zHvIveqvPy)sM_7YLVvoP;)=QXnVH#n!&mdpPIJ8vTwHY6=!~Dm@{CKBXmhgK*#f{* zraM%9C|zQC0b!>rN|abst=;fZrM>ExZWn4@BrWx{$aX*FCM{&CMGUNyAo$rit4E8`UR~7K|X(8@OMM^|i>R!bHx^BaPdtB(+K+vgl)P2VRvEQMDd&%d&$Xkj{ z|Mu;5T2)9^?ikYSYsV{}vb(5X5;}QqfU^C_0EIsXwzr2T&wWTi+vS&>henThpXIPguT`0FP+|$e;e+rc>6phJ9`st zoHKoP(M2ch52Lt!;&#`WllI1CI%0tgVkZaR5&RF8%IrBSZE8$S?`(9d3X5%*9mkr= zsV=CD?$%~I+Lu{0NlSmKVwkx1uOf@|x8)pqSs~>J z@Z6b$;g5jL!xjHprW2@5hO6syoO?K*G`aPrFsTTBrSS zr@dSAnF1vAaQBmEIhbk~a?TfBcqcd>|5s$;CGz(#a)WluFn*LdvAHtfIZ_h`Z3BlK z0lZ*|V@J-pfIGu9J$GHF>Z;r?o5}Q_3U;2Evs8ZF_QLl3Rj9H#4pla^ht(Z62SHIY z=+hWqOZFw#%l$-l{IR#;I2oBNTOT)|{%x)WK`o9$Gf$Y$J$~!XP()hduj%{2w}pMZ zLWKnR{-z{!MFp7;Qd(($zdC%3xn^cLr`Jc}GKUo4EoC@GsA(b)pQ$QJI_xy7s+t8Mj!n$(bnRJ-KGWoQy#^cJ!2$guXf;2vBcZU+I zN;mo2zkc&gDTa+ff|OLGI1+2s^(Q;j7GAtOrLLUHhiZH=5b2g?ScqR!-#YI+r8sRa zxH5_6E$-|3E00m@ygw)U!01w>&-@j{)8d&R{0x{)@V6M>*`e!CLa_|s4t1`39_Tk` z8uDATU47H>l7zJ?7tP~%cAk@x48r=Dl0g~fd%gCgyd2#r*JNw|Ig3XQYJ61=fu^=K z+=x~43s#f_kRAIy8B}17|I;pA|6lDy$`W2qel9IDdC}TLG^iOPMMljj+aUDNPCvF5&Ft)y(L$6 zZ2L_^%##7RVF1mn@~55@Y(Nln5%Uf1GYS+&NkUNnT}tOZq8pYp7e|%=2TqN@^zG>` zQluU=pW1=XipJ6rFe!%dwZnHeYgmfC(Y(PNKd7lF9XE#sdKE31svivVnQBoN&w0M^Jg_eb3!s7f zZEjFe`WD#;oyp5+M$o@2imyJ-jIeg^4zpm721`KzA2#S}iH~z+XKuMu7ayH|%G}@- zkCmNnyfy%B9r6J^MD-+ICG{C&+|K@GZ_ASnI!t|`wA7QWg)cQ2LEOV*jq z4GkB6P{0PA-VkJu5(2CG=I*7B*7{95aE-Nd9bAZh(0?_s^_xhOV|Fyaz4-3LGCBwD z(#w-BjhCH=_YPV(Zn`=@Dv?BcCt3$TGY=q)TUi=}yUyE?-{3z$CvC8&3<|uF$tFVr zsa4jGT!LnC6HjjLPLjTsJ}<0((ps23dZF%#)}8+@#qPJ0&V%ie-NmVGOs=0z1QX#_ zN=URzMg+qHw^wE|eW{*K%WXfV#i;Awe>qD-zc)(vrth+NUu#5-z15JiQ@?z_QQixq z`6fQqC!x3E-5i&Uh9oQZr{myP<=Iy*($rX64c&I?SLnl<{4kntd@&Uay2QKb&pTK> z^j-Hpw5Q5WtfyslUq5xdj8m$;&%{LawYe)IA2BvFV0y+XEHx|3 z-z&cjt)lUGH0uZmtebV|5p2e6<`H{HoeyAoM`zy=W}MH=Rub09}nB z^FyKn9Ven?dXpPBFUak>4Gtfu50{Ynz%&7DZ!Y)TR!$~vL6I!HQYjWf_o;XP+9>mY zSGa@YxT@WQhTqm7s18;jbVAD@2-bC=T9qr5OCCi{xoIB0h!T$aJhv3j-K*9n5x2aQ zNOC_Ww^u0n?lcQwyMQjC0Jo(^S;HHYEHqBbMAmGyJRkYs?(f`O658B6HYCuYEl7** z<)m;yC#}fS(2i4=?-KdvHz@rj$&lra0Hfc~$Z@fsFu&LS9M(GN0J=~zenN-Cdgv<9 zKS!U!KD&dW8RU~oxLh)u^$vb5xw=#OGx-jfOf(dL8OQbAt&6=u=10WC6-t{XhZp0@ z=5%Vo_%G)EmvTmStXlowLxPl-v;KpZ$xP8vn-Dsk)+CPNqAG}q@Az_~H{ni__vH3+ zQ!ZFuN;KDaG|k0;`&!Q$*@?B*jHMpLe&+enu5}|Lb!CJT8(a*#hD5`jm`PNm-AtEZ zw`AI@oan(;n#Lwhon1F9t5N$Z)3?#z7re%F?afeP{rO{p>N%E{{8C1oNeTSll`9kP zTi9|$b_y7(%K5fPOA4H~9EIp}$k*IMF8(^Q=<_3BH5OBZz}uZY0(PD4p*avk{QiQz z03B_szH|gUAFQMBLR%4|(Bvh$Iibg{wjBZ88VVF2sUslSw&w^K^L04_PCS4#(g*1O z|9*vP0AdB0rsQ#BisGVJW<#3>QcL{K9pUWk#3UVid!Ul`q_${&@V<;yYKauF2018a zs^lcgp8BxOw|a2Byf%MQvHx3>sZwf5Ur32mlhnX^xyW2 z6FU^2eY@LNe~5Xpt`SfZKhhAO)jJpG-21BLL(Y;<+mA1I4CHCInXa#5DjRNa$;c1B z;yK}^e4xjEugAJ)vhP6f3Q_)%EvJ(;DLdR#ELVw?=5Dk9P_hf(uvteF)8~O`R6KW4 zRrbQ=pn4MQB6Hj(J)J=NF2N`t z9hP2`+VL4IzRu|MMgetPT1M&?+#K&JHGBhZ8^n+8$0T)aG1fFq+89jCc_!om2Bti0?)ZgIjfHJGmVJ#Yu zm_h%XcnX^u1;r%ECmnj12qqcnTFPKC&V63g7OY>+NLpkl=sv9ux>TaX?|09IR)zVa z11drPUQ$~PrdPo8#vX>{Gd4Vcw!zoIS1Y%AW9nl4g = { Promptguard: "promptguard", LlmAsAJudge: "llm_as_a_judge", Xecguard: "xecguard", + QostodianNexus: "qostodian_nexus", }; // Function to populate provider map from API response - updates the original map @@ -138,6 +139,7 @@ export const guardrailLogoMap: Record = { "LiteLLM Content Filter": `${asset_logos_folder}litellm_logo.jpg`, "LiteLLM LLM as a Judge": `${asset_logos_folder}litellm_logo.jpg`, "Akto": `${asset_logos_folder}akto.svg`, + "Qostodian Nexus": `${asset_logos_folder}qohash.jpg`, }; export const getGuardrailLogoAndName = (guardrailValue: string): { logo: string; displayName: string } => { From 1b6914d44c52a3884aff412ef417ec52bee9141f Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Thu, 2 Apr 2026 05:08:57 +0200 Subject: [PATCH 04/29] fix(cost): pass service_tier through azure and azure_ai cost calculation (#24926) service_tier (priority/flex) was not forwarded to generic_cost_per_token for azure and azure_ai providers, so tier-specific pricing was ignored and standard pricing was always returned. Other providers (openai, bedrock, gemini, vertex_ai) already pass it correctly. --- litellm/cost_calculator.py | 4 +- litellm/llms/azure/cost_calculation.py | 4 +- litellm/llms/azure_ai/cost_calculator.py | 2 + .../llms/azure/test_azure_cost_calculation.py | 75 +++++++++++++++++++ .../azure_ai/test_azure_ai_cost_calculator.py | 48 ++++++++++++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/azure/test_azure_cost_calculation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8a68d74be5..4d7b44bfbd 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -513,7 +513,8 @@ def cost_per_token( # noqa: PLR0915 return fireworks_ai_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "azure": return azure_openai_cost_per_token( - model=model, usage=usage_block, response_time_ms=response_time_ms + model=model, usage=usage_block, response_time_ms=response_time_ms, + service_tier=service_tier, ) elif custom_llm_provider == "gemini": return gemini_cost_per_token( @@ -539,6 +540,7 @@ def cost_per_token( # noqa: PLR0915 usage=usage_block, response_time_ms=response_time_ms, request_model=request_model, + service_tier=service_tier, ) else: model_info = _cached_get_model_info_helper( diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 5b411095ea..2e2e0c4965 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -12,7 +12,8 @@ from litellm.utils import get_model_info def cost_per_token( - model: str, usage: Usage, response_time_ms: Optional[float] = 0.0 + model: str, usage: Usage, response_time_ms: Optional[float] = 0.0, + service_tier: Optional[str] = None, ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -47,4 +48,5 @@ def cost_per_token( model=model, usage=usage, custom_llm_provider="azure", + service_tier=service_tier, ) diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 067181b946..755d44fdef 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -65,6 +65,7 @@ def cost_per_token( usage: Usage, response_time_ms: Optional[float] = 0.0, request_model: Optional[str] = None, + service_tier: Optional[str] = None, ) -> Tuple[float, float]: """ Calculate the cost per token for Azure AI models. @@ -102,6 +103,7 @@ def cost_per_token( model=model, usage=usage, custom_llm_provider="azure_ai", + service_tier=service_tier, ) except Exception as e: # For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map diff --git a/tests/test_litellm/llms/azure/test_azure_cost_calculation.py b/tests/test_litellm/llms/azure/test_azure_cost_calculation.py new file mode 100644 index 0000000000..53c91032b3 --- /dev/null +++ b/tests/test_litellm/llms/azure/test_azure_cost_calculation.py @@ -0,0 +1,75 @@ +""" +Test Azure OpenAI cost calculator — service_tier pricing. +""" + +import pytest + +import litellm +from litellm.llms.azure.cost_calculation import cost_per_token +from litellm.types.utils import Usage + + +# Register a test model with tier-specific pricing +TEST_MODEL = "test-azure-gpt-4.1" +TEST_MODEL_COST = { + TEST_MODEL: { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "input_cost_per_token_priority": 0.01, + "output_cost_per_token_priority": 0.02, + "input_cost_per_token_flex": 0.0005, + "output_cost_per_token_flex": 0.001, + "litellm_provider": "azure", + "max_tokens": 8192, + } +} + + +class TestAzureServiceTierCostCalculation: + """Test that service_tier is passed through Azure cost calculation.""" + + @pytest.fixture(autouse=True) + def register_test_model(self): + litellm.register_model(model_cost=TEST_MODEL_COST) + + def test_service_tier_priority_higher_cost(self): + """Priority tier should cost more than standard.""" + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + standard_prompt, standard_completion = cost_per_token( + model=TEST_MODEL, usage=usage + ) + priority_prompt, priority_completion = cost_per_token( + model=TEST_MODEL, usage=usage, service_tier="priority" + ) + + assert priority_prompt > standard_prompt + assert priority_completion > standard_completion + + def test_service_tier_flex_lower_cost(self): + """Flex tier should cost less than standard.""" + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + standard_prompt, standard_completion = cost_per_token( + model=TEST_MODEL, usage=usage + ) + flex_prompt, flex_completion = cost_per_token( + model=TEST_MODEL, usage=usage, service_tier="flex" + ) + + assert flex_prompt < standard_prompt + assert flex_completion < standard_completion + + def test_service_tier_none_returns_standard(self): + """service_tier=None should return standard pricing.""" + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + none_prompt, none_completion = cost_per_token( + model=TEST_MODEL, usage=usage, service_tier=None + ) + standard_prompt, standard_completion = cost_per_token( + model=TEST_MODEL, usage=usage, service_tier="standard" + ) + + assert abs(none_prompt - standard_prompt) < 1e-10 + assert abs(none_completion - standard_completion) < 1e-10 diff --git a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py index 37add41b83..20260c744f 100644 --- a/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/test_litellm/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -451,3 +451,51 @@ class TestAzureModelRouterCostBreakdown: assert logging_obj.cost_breakdown["additional_costs"][ "Azure Model Router Flat Cost" ] == pytest.approx(expected_flat_cost, rel=1e-9) + + +class TestAzureAIServiceTierCostCalculation: + """Test that service_tier is passed through Azure AI cost calculation.""" + + @pytest.fixture(autouse=True) + def register_test_model(self): + import litellm + litellm.register_model(model_cost={ + "test-azure-ai-model": { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "input_cost_per_token_priority": 0.01, + "output_cost_per_token_priority": 0.02, + "input_cost_per_token_flex": 0.0005, + "output_cost_per_token_flex": 0.001, + "litellm_provider": "azure_ai", + "max_tokens": 8192, + } + }) + + def test_service_tier_priority_higher_cost(self): + """Priority tier should cost more than standard for azure_ai.""" + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + standard_prompt, standard_completion = cost_per_token( + model="test-azure-ai-model", usage=usage + ) + priority_prompt, priority_completion = cost_per_token( + model="test-azure-ai-model", usage=usage, service_tier="priority" + ) + + assert priority_prompt > standard_prompt + assert priority_completion > standard_completion + + def test_service_tier_flex_lower_cost(self): + """Flex tier should cost less than standard for azure_ai.""" + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + standard_prompt, standard_completion = cost_per_token( + model="test-azure-ai-model", usage=usage + ) + flex_prompt, flex_completion = cost_per_token( + model="test-azure-ai-model", usage=usage, service_tier="flex" + ) + + assert flex_prompt < standard_prompt + assert flex_completion < standard_completion From 7e58c7139a46615aeed176742b1d1afcfcdedef4 Mon Sep 17 00:00:00 2001 From: milan-berri Date: Thu, 2 Apr 2026 05:10:36 +0200 Subject: [PATCH 05/29] fix(proxy): include team membership budget in combined_view for RPM/TPM (#24925) Join LiteLLM_BudgetTable as b_tm on team membership budget_id and select team_member_tpm_limit / team_member_rpm_limit so virtual key auth populates limits for parallel_request_limiter_v3. Add test_team_member_rate_limits_v3_raises_429_when_over_limit mirroring existing key-level OVER_LIMIT / HTTP 429 coverage. Made-with: Cursor --- litellm/proxy/utils.py | 3 + .../hooks/test_parallel_request_limiter_v3.py | 77 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d2dfa17751..31dab435b6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3187,6 +3187,8 @@ class PrismaClient: t.organization_id as org_id, p.project_alias AS project_alias, tm.spend AS team_member_spend, + b_tm.tpm_limit AS team_member_tpm_limit, + b_tm.rpm_limit AS team_member_rpm_limit, m.aliases AS team_model_aliases, -- Added comma to separate b.* columns b.max_budget AS litellm_budget_table_max_budget, @@ -3202,6 +3204,7 @@ class PrismaClient: FROM "LiteLLM_VerificationToken" AS v LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id LEFT JOIN "LiteLLM_TeamMembership" AS tm ON v.team_id = tm.team_id AND tm.user_id = v.user_id + LEFT JOIN "LiteLLM_BudgetTable" AS b_tm ON tm.budget_id = b_tm.budget_id LEFT JOIN "LiteLLM_ModelTable" m ON t.model_id = m.id LEFT JOIN "LiteLLM_BudgetTable" AS b ON v.budget_id = b.budget_id LEFT JOIN "LiteLLM_ProjectTable" AS p ON v.project_id = p.project_id diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index d4fb5b7271..370477c360 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -1029,6 +1029,83 @@ async def test_team_member_rate_limits_v3(): ), "Team member TPM limit should be set" +@pytest.mark.asyncio +async def test_team_member_rate_limits_v3_raises_429_when_over_limit(): + """ + When should_rate_limit reports OVER_LIMIT for the team_member descriptor, the + pre-call hook raises HTTP 429 with rate_limit headers — same contract as + test_rpm_api_key_rate_limits_v3 / test_tpm_api_key_rate_limits_v3. + """ + _api_key = hash_token("sk-12345") + _team_id = "team_123" + _user_id = "user_456" + + user_api_key_dict = UserAPIKeyAuth( + api_key=_api_key, + team_id=_team_id, + user_id=_user_id, + team_member_rpm_limit=10, + team_member_tpm_limit=1000, + ) + + local_cache = DualCache() + parallel_request_handler = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + + captured_descriptors = None + + async def mock_should_rate_limit(descriptors, **kwargs): + nonlocal captured_descriptors + captured_descriptors = descriptors + return { + "overall_code": "OVER_LIMIT", + "statuses": [ + { + "code": "OVER_LIMIT", + "current_limit": 10, + "limit_remaining": -1, + "rate_limit_type": "requests", + "descriptor_key": "team_member", + }, + { + "code": "OK", + "current_limit": 1000, + "limit_remaining": 500, + "rate_limit_type": "tokens", + "descriptor_key": "team_member", + }, + ], + } + + parallel_request_handler.should_rate_limit = mock_should_rate_limit + + error = None + try: + await parallel_request_handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + except HTTPException as e: + error = e + assert e.status_code == 429 + assert "rate_limit_type" in e.headers + assert e.headers.get("rate_limit_type") == "requests" + assert "retry-after" in e.headers + + assert error is not None, "An Exception must be thrown" + assert captured_descriptors is not None, "Rate limit descriptors should be captured" + team_member_descriptor = None + for descriptor in captured_descriptors: + if descriptor["key"] == "team_member": + team_member_descriptor = descriptor + break + assert team_member_descriptor is not None + assert team_member_descriptor["value"] == f"{_team_id}:{_user_id}" + + @pytest.mark.asyncio async def test_dynamic_rate_limiting_v3(): """ From 9d6983c4c0bc85d94ffd98dfd17e774df8827bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?d=20=F0=9F=94=B9?= Date: Thu, 2 Apr 2026 11:13:15 +0800 Subject: [PATCH 06/29] fix(gemini): handle Gemini Files API URIs without fetching (#24922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gemini): handle Gemini Files API URIs without fetching Fixes #24907 When a file is uploaded via the Gemini Files API, the returned URI (https://generativelanguage.googleapis.com/v1beta/files/...) starts with 'https://' and hits the generic HTTPS handler in _process_gemini_media(). That handler calls _get_image_mime_type_from_url() which tries to fetch the URL — but Gemini Files API URLs return 403 when accessed directly, causing: 'Unable to determine mime type for file_id: ...' Fix: add an early elif that matches Gemini Files API URLs and passes them through as file_data without trying to fetch the URL. When an explicit format is provided it's included; otherwise the Gemini API infers the MIME type from its stored metadata. Exactly matches the fix direction suggested by the issue reporter (rodriciru). * fix: anchor Gemini Files API URL check with startswith Address greptile P2: replace `in` substring check with `startswith` to prevent query-string injection bypass (e.g. `https://evil.com/?ref=https://generativelanguage...`). Also adds trailing slash to match only valid file URIs. --------- Co-authored-by: voidborne-d --- .../llms/vertex_ai/gemini/transformation.py | 15 ++++++ .../test_vertex_ai_gemini_transformation.py | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 87bd484382..0b4f900ef8 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -212,6 +212,21 @@ def _process_gemini_media( return _apply_gemini_metadata( part, model, media_resolution_enum, video_metadata ) + elif image_url.startswith( + "https://generativelanguage.googleapis.com/v1beta/files/" + ): + # Gemini Files API URIs — the file is already uploaded to Google's + # servers; pass the URI through as file_data without fetching it. + # These URLs return 403 when accessed directly, so we must not try + # to resolve their MIME type via HTTP. + if format: + file_data = FileDataType(mime_type=format, file_uri=image_url) + else: + file_data = FileDataType(file_uri=image_url) + part = {"file_data": file_data} + return _apply_gemini_3_metadata( + part, model, media_resolution_enum, video_metadata + ) elif ( "https://" in image_url and (image_type := format or _get_image_mime_type_from_url(image_url)) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 831d1ef464..780e7d2ba9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -1291,6 +1291,53 @@ def test_file_data_field_order_gcs_urls(): ), "mime_type must come before file_uri in the file_data dict" +def test_gemini_files_api_uri_without_format(): + """ + Test that Gemini Files API URIs work WITHOUT an explicit format/mime_type. + + When a user uploads a file via the Gemini Files API and then references it + by URI (https://generativelanguage.googleapis.com/v1beta/files/...), + the file is already on Google's servers. These URLs return 403 when + fetched directly, so _process_gemini_media must NOT try to resolve the + MIME type via HTTP. Instead it should pass the URI through as file_data + and let the Gemini API resolve the type from its stored metadata. + + Related issue: https://github.com/BerriAI/litellm/issues/24907 + """ + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + file_url = "https://generativelanguage.googleapis.com/v1beta/files/37eh7rsw1vfe" + + # Should NOT raise — previously this hit the generic https:// handler + # which called _get_image_mime_type_from_url() and got a 403. + result = _process_gemini_media(image_url=file_url) + + assert "file_data" in result + file_data = result["file_data"] + assert file_data["file_uri"] == file_url + # When no format is provided, mime_type should be absent so the + # Gemini API infers it from the stored file metadata. + assert "mime_type" not in file_data + + +def test_gemini_files_api_uri_with_format(): + """ + Test that Gemini Files API URIs correctly forward an explicit format. + + Related issue: https://github.com/BerriAI/litellm/issues/24907 + """ + from litellm.llms.vertex_ai.gemini.transformation import _process_gemini_media + + file_url = "https://generativelanguage.googleapis.com/v1beta/files/n1vhxa28lyaw" + + result = _process_gemini_media(image_url=file_url, format="text/plain") + + assert "file_data" in result + file_data = result["file_data"] + assert file_data["file_uri"] == file_url + assert file_data["mime_type"] == "text/plain" + + def test_extract_file_data_with_path_object(): """ Test that filename is correctly extracted from Path objects for MIME type detection. From b8f5189b65f220dc9f038ee899515c85bc583d8f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 2 Apr 2026 08:45:06 +0530 Subject: [PATCH 07/29] fix(azure): forward api_version to aembedding() for Azure AI Foundry v1 endpoints (#24911) When aembedding=True, api_version was not passed to self.aembedding(), causing get_azure_openai_client() to receive None instead of "v1". This made _is_azure_v1_api_version() return False, so AsyncAzureOpenAI was selected instead of AsyncOpenAI, constructing the wrong request URL and returning 404. Fixes #24848 Co-authored-by: Claude Sonnet 4.6 --- litellm/llms/azure/azure.py | 1 + tests/litellm/llms/azure/__init__.py | 0 .../llms/azure/test_azure_embedding.py | 94 +++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 tests/litellm/llms/azure/__init__.py create mode 100644 tests/litellm/llms/azure/test_azure_embedding.py diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 61cfd54b56..ca370366b6 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -792,6 +792,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): client=client, litellm_params=litellm_params, api_base=api_base, + api_version=api_version, ) azure_client = self.get_azure_openai_client( api_version=api_version, diff --git a/tests/litellm/llms/azure/__init__.py b/tests/litellm/llms/azure/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/litellm/llms/azure/test_azure_embedding.py b/tests/litellm/llms/azure/test_azure_embedding.py new file mode 100644 index 0000000000..22ee503ef0 --- /dev/null +++ b/tests/litellm/llms/azure/test_azure_embedding.py @@ -0,0 +1,94 @@ +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from litellm.llms.azure.azure import AzureChatCompletion +from litellm.types.utils import EmbeddingResponse, Usage + + +def _make_embedding_response() -> EmbeddingResponse: + return EmbeddingResponse( + model="text-embedding-3-large", + usage=Usage(prompt_tokens=3, completion_tokens=0, total_tokens=3), + data=[{"embedding": [0.1, 0.2, 0.3], "index": 0, "object": "embedding"}], + ) + + +def _make_logging_obj() -> MagicMock: + return MagicMock() + + +class TestAzureV1AsyncEmbedding: + def test_aembedding_receives_api_version(self): + """Regression: api_version must be forwarded to aembedding() when aembedding=True. + Without the fix, it was silently dropped, causing AsyncAzureOpenAI to be used + instead of AsyncOpenAI for Azure AI Foundry (v1) endpoints. Fixes #24848.""" + handler = AzureChatCompletion() + + with patch.object(handler, "aembedding") as mock_aembedding: + handler.embedding( + model="text-embedding-3-large", + input=["hello world"], + api_base="https://my-endpoint.openai.azure.com", + api_version="v1", + timeout=60.0, + logging_obj=_make_logging_obj(), + model_response=_make_embedding_response(), + optional_params={}, + api_key="fake-key", + aembedding=True, + litellm_params={}, + ) + + mock_aembedding.assert_called_once() + _, kwargs = mock_aembedding.call_args + assert kwargs.get("api_version") == "v1" + + def test_get_azure_openai_client_returns_async_openai_for_v1(self): + from openai import AsyncAzureOpenAI, AsyncOpenAI + + handler = AzureChatCompletion() + client = handler.get_azure_openai_client( + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="v1", + _is_async=True, + litellm_params={}, + ) + + assert isinstance(client, AsyncOpenAI) + assert not isinstance(client, AsyncAzureOpenAI) + + def test_get_azure_openai_client_uses_v1_base_url(self): + handler = AzureChatCompletion() + client = handler.get_azure_openai_client( + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="v1", + _is_async=True, + litellm_params={}, + ) + + assert client is not None + assert "/openai/v1/" in str(client.base_url) + + @pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) + def test_all_v1_variants_use_openai_client(self, api_version: str): + from openai import AsyncOpenAI + + handler = AzureChatCompletion() + client = handler.get_azure_openai_client( + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version=api_version, + _is_async=True, + litellm_params={}, + ) + + assert isinstance(client, AsyncOpenAI) From 9b90fc07d2cf41f1e59f8d059222f7b430ca197d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 5 Apr 2026 01:14:54 -0700 Subject: [PATCH 08/29] chore: fixes From b0ac74c55614582bd53a06da9828ab9cc1f9f061 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 16 Apr 2026 18:46:45 +0530 Subject: [PATCH 09/29] Fix import error --- litellm/_logging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 73a0846983..96035227f9 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,10 +1,11 @@ import ast import logging import os +import re import sys from datetime import datetime from logging import Formatter -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -21,6 +22,8 @@ _ENABLE_SECRET_REDACTION = ( os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true" ) +_REDACTED = "REDACTED" + def _build_secret_patterns() -> re.Pattern: From b3fdb5cc6998d4285d01be0abc6a1b3147b89797 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 16 Apr 2026 19:09:35 +0530 Subject: [PATCH 10/29] Fix import error --- litellm/_logging.py | 1 - litellm/llms/vertex_ai/gemini/transformation.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 96035227f9..8216777645 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List, Optional from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.secret_redaction import redact_string set_verbose = False diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 0b4f900ef8..4d1b714db9 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -222,7 +222,8 @@ def _process_gemini_media( if format: file_data = FileDataType(mime_type=format, file_uri=image_url) else: - file_data = FileDataType(file_uri=image_url) + # Gemini Files API references can be passed through as URI-only. + file_data = cast(FileDataType, {"file_uri": image_url}) part = {"file_data": file_data} return _apply_gemini_3_metadata( part, model, media_resolution_enum, video_metadata From a86938a4af4ba5d1774584d72addcf8729dbd535 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 16 Apr 2026 19:20:41 +0530 Subject: [PATCH 11/29] Fix import error --- litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py index 7c856b676b..7de15192d6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py @@ -4,6 +4,7 @@ Qostodian Nexus (by Qohash) — LiteLLM guardrail integration. import os from typing import TYPE_CHECKING, Literal, Optional, Type +from litellm.integrations.custom_guardrail import log_guardrail_information from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api.generic_guardrail_api import ( GenericGuardrailAPI, ) @@ -44,6 +45,7 @@ class QostodianNexus(GenericGuardrailAPI): **kwargs, ) + @log_guardrail_information async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, From ed853e138f4a818771dee39711eedbbb0ef6d2b7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 16 Apr 2026 19:37:48 +0530 Subject: [PATCH 12/29] Fix code qa --- tests/test_litellm/test_secret_redaction.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 5c9b3b37df..dbc339676e 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -7,6 +7,7 @@ import pytest from litellm._logging import ( JsonFormatter, + _redact_string, _secret_filter, verbose_logger, verbose_proxy_logger, From 319180f6505a9d0e187f39c0e27949bfb15e630e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 17 Apr 2026 18:46:14 +0530 Subject: [PATCH 13/29] Fix ruff errors --- .../litellm_core_utils/exception_mapping_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index cc665231d1..b844b44bb5 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2313,7 +2313,7 @@ def exception_type( # type: ignore # noqa: PLR0915 else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors raise APIConnectionError( - message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}", + message=f"{exception_provider} APIConnectionError - {message}\n{redact_string(traceback.format_exc()) if _ENABLE_SECRET_REDACTION else traceback.format_exc()}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -2440,7 +2440,10 @@ def exception_type( # type: ignore # noqa: PLR0915 else: raise APIConnectionError( message="{}\n{}".format( - str(original_exception), _redact_string(traceback.format_exc()) + str(original_exception), + redact_string(traceback.format_exc()) + if _ENABLE_SECRET_REDACTION + else traceback.format_exc(), ), llm_provider=custom_llm_provider, model=model, @@ -2470,7 +2473,10 @@ def exception_type( # type: ignore # noqa: PLR0915 raise e # it's already mapped raised_exc = APIConnectionError( message="{}\n{}".format( - original_exception, _redact_string(traceback.format_exc()) + original_exception, + redact_string(traceback.format_exc()) + if _ENABLE_SECRET_REDACTION + else traceback.format_exc(), ), llm_provider="", model="", From 8c55c541d390b5a01e646cc5fcfd76c8c770739c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 17 Apr 2026 18:49:19 +0530 Subject: [PATCH 14/29] Fix ruff errors --- litellm/litellm_core_utils/exception_mapping_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index b844b44bb5..2c1d92920a 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2313,7 +2313,7 @@ def exception_type( # type: ignore # noqa: PLR0915 else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors raise APIConnectionError( - message=f"{exception_provider} APIConnectionError - {message}\n{redact_string(traceback.format_exc()) if _ENABLE_SECRET_REDACTION else traceback.format_exc()}", + message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -2441,9 +2441,7 @@ def exception_type( # type: ignore # noqa: PLR0915 raise APIConnectionError( message="{}\n{}".format( str(original_exception), - redact_string(traceback.format_exc()) - if _ENABLE_SECRET_REDACTION - else traceback.format_exc(), + _redact_string(traceback.format_exc()), ), llm_provider=custom_llm_provider, model=model, @@ -2474,9 +2472,7 @@ def exception_type( # type: ignore # noqa: PLR0915 raised_exc = APIConnectionError( message="{}\n{}".format( original_exception, - redact_string(traceback.format_exc()) - if _ENABLE_SECRET_REDACTION - else traceback.format_exc(), + _redact_string(traceback.format_exc()), ), llm_provider="", model="", From caa0db38432d44a36aeb1608468495e97266f4cf Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Mon, 9 Mar 2026 15:20:58 -0700 Subject: [PATCH 15/29] adding crusoe to litellm --- docs/my-website/docs/providers/crusoe.md | 186 ++++++++++++++++++ litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 1 + litellm/constants.py | 4 + .../get_llm_provider_logic.py | 10 + litellm/llms/crusoe/__init__.py | 0 litellm/llms/crusoe/chat/__init__.py | 0 litellm/llms/crusoe/chat/transformation.py | 42 ++++ model_prices_and_context_window.json | 89 +++++++++ tests/llm_translation/test_crusoe.py | 164 +++++++++++++++ 10 files changed, 500 insertions(+) create mode 100644 docs/my-website/docs/providers/crusoe.md create mode 100644 litellm/llms/crusoe/__init__.py create mode 100644 litellm/llms/crusoe/chat/__init__.py create mode 100644 litellm/llms/crusoe/chat/transformation.py create mode 100644 tests/llm_translation/test_crusoe.py diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md new file mode 100644 index 0000000000..696da3fa1b --- /dev/null +++ b/docs/my-website/docs/providers/crusoe.md @@ -0,0 +1,186 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Crusoe + +## Overview + +| Property | Details | +|-------|-------| +| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. | +| Provider Route on LiteLLM | `crusoe/` | +| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) | +| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1/` | +| Supported Operations | [`/chat/completions`](#sample-usage) | + +
+
+ +**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests** + +## Available Models + +| Model | Description | Context Window | +|-------|-------------|----------------| +| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens | +| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens | +| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens | +| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens | +| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens | +| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens | +| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens | + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key +``` + +## Usage - LiteLLM Python SDK + +### Non-streaming + +```python showLineNumbers title="Crusoe Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# Crusoe call +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages +) + +print(response) +``` + +### Streaming + +```python showLineNumbers title="Crusoe Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +messages = [{"content": "Write a short story about AI", "role": "user"}] + +# Crusoe call with streaming +response = completion( + model="crusoe/meta-llama/Llama-3.1-70B-Instruct", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + +### Function Calling + +```python showLineNumbers title="Crusoe Function Calling" +import os +import litellm +from litellm import completion + +os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + } + }, + "required": ["location"] + } + } +}] + +messages = [{"role": "user", "content": "What's the weather in Boston?"}] + +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=messages, + tools=tools, + tool_choice="auto" +) + +print(response) +``` + +## Usage - LiteLLM Proxy Server + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: llama-3.3-70b + litellm_params: + model: crusoe/meta-llama/Llama-3.3-70B-Instruct + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-r1 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-R1-0528 + api_key: os.environ/CRUSOE_API_KEY + - model_name: deepseek-v3 + litellm_params: + model: crusoe/deepseek-ai/DeepSeek-V3-0324 + api_key: os.environ/CRUSOE_API_KEY + - model_name: qwen3-235b + litellm_params: + model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507 + api_key: os.environ/CRUSOE_API_KEY + - model_name: kimi-k2 + litellm_params: + model: crusoe/moonshotai/Kimi-K2-Thinking + api_key: os.environ/CRUSOE_API_KEY +``` + +## Custom API Base + +```python showLineNumbers title="Custom API Base" +import os +import litellm +from litellm import completion + +# Using environment variable +os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1/" +os.environ["CRUSOE_API_KEY"] = "" # your API key + +# Or pass directly +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"content": "Hello!", "role": "user"}], + api_base="https://custom.crusoecloud.com/v1/", + api_key="your-api-key" +) +``` + +## Supported OpenAI Parameters + +- `temperature` +- `max_tokens` +- `max_completion_tokens` +- `top_p` +- `frequency_penalty` +- `presence_penalty` +- `stop` +- `n` +- `stream` +- `tools` +- `tool_choice` +- `response_format` +- `seed` +- `user` +- `logit_bias` +- `logprobs` +- `top_logprobs` diff --git a/litellm/__init__.py b/litellm/__init__.py index 77fa48625d..014a8d75f1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -600,6 +600,7 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() +crusoe_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -847,6 +848,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) + elif value.get("litellm_provider") == "crusoe": + crusoe_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -1082,6 +1085,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "crusoe": crusoe_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 119e62a5b3..c0a8666eb7 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1140,6 +1140,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "CrusoeChatConfig": (".llms.crusoe.chat.transformation", "CrusoeChatConfig"), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 6c889a317b..5e75c0feab 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,6 +610,7 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", + "crusoe", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -768,6 +769,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://managed-inference-api-proxy.crusoecloud.com/v1/", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -823,6 +825,7 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", + "crusoe", "hyperbolic", "vercel_ai_gateway", "aiml", @@ -851,6 +854,7 @@ openai_text_completion_compatible_providers: List = ( "chutes", "v0", "lambda_ai", + "crusoe", "hyperbolic", "wandb", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c0ca6835ee..54450bccba 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -353,6 +353,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://managed-inference-api-proxy.crusoecloud.com/v1/": + custom_llm_provider = "crusoe" + dynamic_api_key = get_secret_str("CRUSOE_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -919,6 +922,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "crusoe": + ( + api_base, + dynamic_api_key, + ) = litellm.CrusoeChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/llms/crusoe/__init__.py b/litellm/llms/crusoe/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/crusoe/chat/__init__.py b/litellm/llms/crusoe/chat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/crusoe/chat/transformation.py b/litellm/llms/crusoe/chat/transformation.py new file mode 100644 index 0000000000..dab9a9f833 --- /dev/null +++ b/litellm/llms/crusoe/chat/transformation.py @@ -0,0 +1,42 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Crusoe's `/v1/chat/completions` +""" + +from typing import Optional, Tuple + +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class CrusoeChatConfig(OpenAILikeChatConfig): + """ + Crusoe is OpenAI-compatible with standard endpoints. + + Docs: https://docs.crusoecloud.com/managed-inference/overview/index.html + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "crusoe" + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("CRUSOE_API_BASE") + or "https://managed-inference-api-proxy.crusoecloud.com/v1/" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("CRUSOE_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + return [ + "messages", + "model", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + ] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bbe13442d6..8a5a455e6b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22109,6 +22109,95 @@ "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, + "crusoe/deepseek-ai/DeepSeek-R1-0528": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 7e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/google/gemma-3-12b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/openai/gpt-oss-120b": { + "input_cost_per_token": 8e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py new file mode 100644 index 0000000000..254d6530a4 --- /dev/null +++ b/tests/llm_translation/test_crusoe.py @@ -0,0 +1,164 @@ +""" +Tests for Crusoe provider integration +""" +import os +from unittest import mock + +import pytest + +import litellm +from litellm import completion +from litellm.llms.crusoe.chat.transformation import CrusoeChatConfig + +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" + + +def test_crusoe_config_initialization(): + """Test CrusoeChatConfig initializes correctly""" + config = CrusoeChatConfig() + assert config.custom_llm_provider == "crusoe" + + +def test_crusoe_get_openai_compatible_provider_info(): + """Test Crusoe provider info retrieval""" + config = CrusoeChatConfig() + + # Test with default values (no env vars set) + with mock.patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == CRUSOE_API_BASE + assert api_key is None + + # Test with environment variables + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "test-key", + "CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1/", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + assert api_base == "https://custom.crusoecloud.com/v1/" + assert api_key == "test-key" + + # Test with explicit parameters (should override env vars) + with mock.patch.dict( + os.environ, + { + "CRUSOE_API_KEY": "env-key", + "CRUSOE_API_BASE": "https://env.crusoecloud.com/v1/", + }, + ): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://param.crusoecloud.com/v1/", "param-key" + ) + assert api_base == "https://param.crusoecloud.com/v1/" + assert api_key == "param-key" + + +def test_get_llm_provider_crusoe(): + """Test that get_llm_provider correctly identifies Crusoe""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with crusoe/model-name format + model, provider, api_key, api_base = get_llm_provider( + "crusoe/meta-llama/Llama-3.3-70B-Instruct" + ) + assert model == "meta-llama/Llama-3.3-70B-Instruct" + assert provider == "crusoe" + + # Test with api_base containing Crusoe endpoint + model, provider, api_key, api_base = get_llm_provider( + "meta-llama/Llama-3.3-70B-Instruct", + api_base=CRUSOE_API_BASE, + ) + assert model == "meta-llama/Llama-3.3-70B-Instruct" + assert provider == "crusoe" + assert api_base == CRUSOE_API_BASE + + +def test_crusoe_in_provider_lists(): + """Test that Crusoe is registered in all necessary provider lists""" + assert "crusoe" in litellm.openai_compatible_providers + assert "crusoe" in litellm.provider_list + assert CRUSOE_API_BASE in litellm.openai_compatible_endpoints + + +def test_crusoe_models_configuration(): + """Test that Crusoe models are configured correctly""" + from litellm import get_model_info + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm.crusoe_models = set() + litellm.add_known_models() + + crusoe_models = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + + for model in crusoe_models: + model_info = get_model_info(model) + assert model_info is not None, f"Model info not found for {model}" + assert model_info.get("litellm_provider") == "crusoe", ( + f"{model} should have crusoe as provider" + ) + assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + + +def test_crusoe_model_list_populated(): + """Test that crusoe_models list is populated correctly""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + litellm.crusoe_models = set() + litellm.add_known_models() + + assert len(litellm.crusoe_models) > 0, "crusoe_models list should not be empty" + + for model in litellm.crusoe_models: + assert model.startswith("crusoe/"), ( + f"Model {model} should start with 'crusoe/'" + ) + + expected_models = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + + for model in expected_models: + assert model in litellm.crusoe_models, ( + f"{model} should be in crusoe_models list" + ) + + +@pytest.mark.asyncio +async def test_crusoe_completion_call(): + """Test completion call with Crusoe provider (requires CRUSOE_API_KEY)""" + if not os.getenv("CRUSOE_API_KEY"): + pytest.skip("CRUSOE_API_KEY not set") + + try: + response = await litellm.acompletion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"role": "user", "content": "Hello, this is a test"}], + max_tokens=10, + ) + assert response.choices[0].message.content + assert response.model + assert response.usage + except Exception as e: + if "crusoe" not in str(e) and "provider" not in str(e).lower(): + raise From d492d8fe820a52879f26613d90bde936c8401fe6 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Thu, 12 Mar 2026 14:15:44 -0700 Subject: [PATCH 16/29] refactor(crusoe): simplify to JSON-based provider registration Replace hand-written CrusoeChatConfig class and manual registrations across constants.py, __init__.py, get_llm_provider_logic.py, and _lazy_imports_registry.py with a single entry in litellm/llms/openai_like/providers.json, consistent with the recommended pattern for OpenAI-compatible providers. Co-Authored-By: Claude Sonnet 4.6 --- litellm/__init__.py | 4 - litellm/_lazy_imports_registry.py | 1 - litellm/constants.py | 4 - .../get_llm_provider_logic.py | 10 -- litellm/llms/crusoe/__init__.py | 0 litellm/llms/crusoe/chat/__init__.py | 0 litellm/llms/crusoe/chat/transformation.py | 42 ------- litellm/llms/openai_like/providers.json | 5 + tests/llm_translation/test_crusoe.py | 93 +++------------ tests/test_litellm/llms/crusoe/test_crusoe.py | 111 ++++++++++++++++++ 10 files changed, 130 insertions(+), 140 deletions(-) delete mode 100644 litellm/llms/crusoe/__init__.py delete mode 100644 litellm/llms/crusoe/chat/__init__.py delete mode 100644 litellm/llms/crusoe/chat/transformation.py create mode 100644 tests/test_litellm/llms/crusoe/test_crusoe.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 014a8d75f1..77fa48625d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -600,7 +600,6 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() -crusoe_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -848,8 +847,6 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) - elif value.get("litellm_provider") == "crusoe": - crusoe_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -1085,7 +1082,6 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, - "crusoe": crusoe_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index c0a8666eb7..119e62a5b3 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1140,7 +1140,6 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), - "CrusoeChatConfig": (".llms.crusoe.chat.transformation", "CrusoeChatConfig"), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 5e75c0feab..6c889a317b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,7 +610,6 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", - "crusoe", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -769,7 +768,6 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", - "https://managed-inference-api-proxy.crusoecloud.com/v1/", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -825,7 +823,6 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", - "crusoe", "hyperbolic", "vercel_ai_gateway", "aiml", @@ -854,7 +851,6 @@ openai_text_completion_compatible_providers: List = ( "chutes", "v0", "lambda_ai", - "crusoe", "hyperbolic", "wandb", ] diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 54450bccba..c0ca6835ee 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -353,9 +353,6 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") - elif endpoint == "https://managed-inference-api-proxy.crusoecloud.com/v1/": - custom_llm_provider = "crusoe" - dynamic_api_key = get_secret_str("CRUSOE_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -922,13 +919,6 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) - elif custom_llm_provider == "crusoe": - ( - api_base, - dynamic_api_key, - ) = litellm.CrusoeChatConfig()._get_openai_compatible_provider_info( - api_base, api_key - ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/llms/crusoe/__init__.py b/litellm/llms/crusoe/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/litellm/llms/crusoe/chat/__init__.py b/litellm/llms/crusoe/chat/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/litellm/llms/crusoe/chat/transformation.py b/litellm/llms/crusoe/chat/transformation.py deleted file mode 100644 index dab9a9f833..0000000000 --- a/litellm/llms/crusoe/chat/transformation.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Translate from OpenAI's `/v1/chat/completions` to Crusoe's `/v1/chat/completions` -""" - -from typing import Optional, Tuple - -from litellm.secret_managers.main import get_secret_str - -from ...openai_like.chat.transformation import OpenAILikeChatConfig - - -class CrusoeChatConfig(OpenAILikeChatConfig): - """ - Crusoe is OpenAI-compatible with standard endpoints. - - Docs: https://docs.crusoecloud.com/managed-inference/overview/index.html - """ - - @property - def custom_llm_provider(self) -> Optional[str]: - return "crusoe" - - def _get_openai_compatible_provider_info( - self, api_base: Optional[str], api_key: Optional[str] - ) -> Tuple[Optional[str], Optional[str]]: - api_base = ( - api_base - or get_secret_str("CRUSOE_API_BASE") - or "https://managed-inference-api-proxy.crusoecloud.com/v1/" - ) # type: ignore - dynamic_api_key = api_key or get_secret_str("CRUSOE_API_KEY") - return api_base, dynamic_api_key - - def get_supported_openai_params(self, model: str) -> list: - return [ - "messages", - "model", - "temperature", - "top_p", - "frequency_penalty", - "presence_penalty", - ] diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 5dd1247001..3ee5a72b4c 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -106,5 +106,10 @@ "base_url": "https://aihubmix.com/v1", "api_key_env": "AIHUBMIX_API_KEY", "api_base_env": "AIHUBMIX_API_BASE" + }, + "crusoe": { + "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1/", + "api_key_env": "CRUSOE_API_KEY", + "api_base_env": "CRUSOE_API_BASE" } } diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py index 254d6530a4..064f4f053b 100644 --- a/tests/llm_translation/test_crusoe.py +++ b/tests/llm_translation/test_crusoe.py @@ -4,24 +4,29 @@ Tests for Crusoe provider integration import os from unittest import mock -import pytest - import litellm -from litellm import completion -from litellm.llms.crusoe.chat.transformation import CrusoeChatConfig CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" -def test_crusoe_config_initialization(): - """Test CrusoeChatConfig initializes correctly""" - config = CrusoeChatConfig() - assert config.custom_llm_provider == "crusoe" +def test_crusoe_json_registry(): + """Test CrusoeChatConfig is loaded from JSON provider registry""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("crusoe") + config = JSONProviderRegistry.get("crusoe") + assert config is not None + assert config.base_url == CRUSOE_API_BASE + assert config.api_key_env == "CRUSOE_API_KEY" + assert config.api_base_env == "CRUSOE_API_BASE" def test_crusoe_get_openai_compatible_provider_info(): """Test Crusoe provider info retrieval""" - config = CrusoeChatConfig() + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() # Test with default values (no env vars set) with mock.patch.dict(os.environ, {}, clear=True): @@ -67,22 +72,6 @@ def test_get_llm_provider_crusoe(): assert model == "meta-llama/Llama-3.3-70B-Instruct" assert provider == "crusoe" - # Test with api_base containing Crusoe endpoint - model, provider, api_key, api_base = get_llm_provider( - "meta-llama/Llama-3.3-70B-Instruct", - api_base=CRUSOE_API_BASE, - ) - assert model == "meta-llama/Llama-3.3-70B-Instruct" - assert provider == "crusoe" - assert api_base == CRUSOE_API_BASE - - -def test_crusoe_in_provider_lists(): - """Test that Crusoe is registered in all necessary provider lists""" - assert "crusoe" in litellm.openai_compatible_providers - assert "crusoe" in litellm.provider_list - assert CRUSOE_API_BASE in litellm.openai_compatible_endpoints - def test_crusoe_models_configuration(): """Test that Crusoe models are configured correctly""" @@ -91,9 +80,6 @@ def test_crusoe_models_configuration(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.crusoe_models = set() - litellm.add_known_models() - crusoe_models = [ "crusoe/meta-llama/Llama-3.3-70B-Instruct", "crusoe/deepseek-ai/DeepSeek-R1-0528", @@ -111,54 +97,3 @@ def test_crusoe_models_configuration(): f"{model} should have crusoe as provider" ) assert model_info.get("mode") == "chat", f"{model} should be in chat mode" - - -def test_crusoe_model_list_populated(): - """Test that crusoe_models list is populated correctly""" - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - litellm.crusoe_models = set() - litellm.add_known_models() - - assert len(litellm.crusoe_models) > 0, "crusoe_models list should not be empty" - - for model in litellm.crusoe_models: - assert model.startswith("crusoe/"), ( - f"Model {model} should start with 'crusoe/'" - ) - - expected_models = [ - "crusoe/meta-llama/Llama-3.3-70B-Instruct", - "crusoe/deepseek-ai/DeepSeek-R1-0528", - "crusoe/deepseek-ai/DeepSeek-V3-0324", - "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", - "crusoe/moonshotai/Kimi-K2-Thinking", - "crusoe/openai/gpt-oss-120b", - "crusoe/google/gemma-3-12b-it", - ] - - for model in expected_models: - assert model in litellm.crusoe_models, ( - f"{model} should be in crusoe_models list" - ) - - -@pytest.mark.asyncio -async def test_crusoe_completion_call(): - """Test completion call with Crusoe provider (requires CRUSOE_API_KEY)""" - if not os.getenv("CRUSOE_API_KEY"): - pytest.skip("CRUSOE_API_KEY not set") - - try: - response = await litellm.acompletion( - model="crusoe/meta-llama/Llama-3.3-70B-Instruct", - messages=[{"role": "user", "content": "Hello, this is a test"}], - max_tokens=10, - ) - assert response.choices[0].message.content - assert response.model - assert response.usage - except Exception as e: - if "crusoe" not in str(e) and "provider" not in str(e).lower(): - raise diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py new file mode 100644 index 0000000000..6ca7317804 --- /dev/null +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -0,0 +1,111 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) +from unittest.mock import patch + +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" + + +def test_crusoe_json_registry(): + """Test Crusoe is registered in the JSON provider registry""" + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("crusoe") + config = JSONProviderRegistry.get("crusoe") + assert config is not None + assert config.base_url == CRUSOE_API_BASE + assert config.api_key_env == "CRUSOE_API_KEY" + assert config.api_base_env == "CRUSOE_API_BASE" + + +def test_crusoe_dynamic_config_defaults(): + """Test dynamic config returns correct default API base""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict(os.environ, {}, clear=True): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + + assert api_base == CRUSOE_API_BASE + assert api_key is None + + +def test_crusoe_dynamic_config_env_vars(): + """Test dynamic config reads CRUSOE_API_KEY and CRUSOE_API_BASE from env""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict( + os.environ, + {"CRUSOE_API_KEY": "test-key", "CRUSOE_API_BASE": "https://custom.crusoe.com/v1/"}, + ): + api_base, api_key = config._get_openai_compatible_provider_info(None, None) + + assert api_base == "https://custom.crusoe.com/v1/" + assert api_key == "test-key" + + +def test_crusoe_dynamic_config_explicit_params(): + """Test explicit params override env vars""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + + with patch.dict(os.environ, {"CRUSOE_API_KEY": "env-key"}): + api_base, api_key = config._get_openai_compatible_provider_info( + "https://override.crusoe.com/v1/", "override-key" + ) + + assert api_base == "https://override.crusoe.com/v1/" + assert api_key == "override-key" + + +def test_crusoe_supported_params(): + """Test dynamic config returns standard OpenAI params""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + params = config.get_supported_openai_params(model="meta-llama/Llama-3.3-70B-Instruct") + + assert isinstance(params, list) + assert len(params) > 0 + assert "temperature" in params + assert "max_tokens" in params + assert "stream" in params + + +def test_crusoe_provider_detection_by_prefix(): + """Test crusoe/model prefix is correctly routed""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, _, _ = get_llm_provider("crusoe/meta-llama/Llama-3.3-70B-Instruct") + assert provider == "crusoe" + assert model == "meta-llama/Llama-3.3-70B-Instruct" + + +def test_crusoe_model_list_populated(): + """Test Crusoe models are present in model_prices_and_context_window.json""" + import litellm + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + expected = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + for model in expected: + assert model in litellm.model_cost, f"{model} not found in model_cost" + assert litellm.model_cost[model].get("litellm_provider") == "crusoe" From 6ae7929d7c0d8577fc2b5f4d78b013cb1036f4c4 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Thu, 12 Mar 2026 14:26:29 -0700 Subject: [PATCH 17/29] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- tests/test_litellm/llms/crusoe/test_crusoe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 6ca7317804..899dab256e 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -1,7 +1,6 @@ import os import sys -sys.path.insert(0, os.path.abspath("../../../../..")) from unittest.mock import patch CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" From 9d64cc8ff8538f01269ae9bec5898d0bf49cedcb Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 12 Mar 2026 18:51:15 -0700 Subject: [PATCH 18/29] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/llms/openai_like/providers.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 3ee5a72b4c..acbbd8b612 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -108,7 +108,7 @@ "api_base_env": "AIHUBMIX_API_BASE" }, "crusoe": { - "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1/", + "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1", "api_key_env": "CRUSOE_API_KEY", "api_base_env": "CRUSOE_API_BASE" } From d7313496f33dd204994cf33f2ea011c1b3887ddc Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 13 Mar 2026 14:13:07 -0700 Subject: [PATCH 19/29] fix: remove trailing slash from CRUSOE_API_BASE and unused sys import --- tests/llm_translation/test_crusoe.py | 2 +- tests/test_litellm/llms/crusoe/test_crusoe.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py index 064f4f053b..66854b2869 100644 --- a/tests/llm_translation/test_crusoe.py +++ b/tests/llm_translation/test_crusoe.py @@ -6,7 +6,7 @@ from unittest import mock import litellm -CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1" def test_crusoe_json_registry(): diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 899dab256e..485c00b357 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -1,9 +1,7 @@ import os -import sys - from unittest.mock import patch -CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1/" +CRUSOE_API_BASE = "https://managed-inference-api-proxy.crusoecloud.com/v1" def test_crusoe_json_registry(): From 9039eb1898517b6e7c0db1480dbe12c0ceaa81a2 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 20 Mar 2026 09:15:19 -0700 Subject: [PATCH 20/29] fix(crusoe): fix docs trailing slash, test state pollution, missing __init__.py - Remove trailing slash from docs Base URL to match providers.json - Wrap model_cost mutations in try/finally to prevent test state leakage - Add missing __init__.py to crusoe test package --- docs/my-website/docs/providers/crusoe.md | 2 +- tests/llm_translation/test_crusoe.py | 29 ++++++++++----- tests/test_litellm/llms/crusoe/__init__.py | 0 tests/test_litellm/llms/crusoe/test_crusoe.py | 37 ++++++++++++------- 4 files changed, 43 insertions(+), 25 deletions(-) create mode 100644 tests/test_litellm/llms/crusoe/__init__.py diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md index 696da3fa1b..dbcd026498 100644 --- a/docs/my-website/docs/providers/crusoe.md +++ b/docs/my-website/docs/providers/crusoe.md @@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem'; | Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. | | Provider Route on LiteLLM | `crusoe/` | | Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) | -| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1/` | +| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` | | Supported Operations | [`/chat/completions`](#sample-usage) |
diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py index 66854b2869..e570e314cd 100644 --- a/tests/llm_translation/test_crusoe.py +++ b/tests/llm_translation/test_crusoe.py @@ -77,10 +77,13 @@ def test_crusoe_models_configuration(): """Test that Crusoe models are configured correctly""" from litellm import get_model_info - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + original_model_cost = litellm.model_cost + original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") - crusoe_models = [ + crusoe_models = [ "crusoe/meta-llama/Llama-3.3-70B-Instruct", "crusoe/deepseek-ai/DeepSeek-R1-0528", "crusoe/deepseek-ai/DeepSeek-V3-0324", @@ -90,10 +93,16 @@ def test_crusoe_models_configuration(): "crusoe/google/gemma-3-12b-it", ] - for model in crusoe_models: - model_info = get_model_info(model) - assert model_info is not None, f"Model info not found for {model}" - assert model_info.get("litellm_provider") == "crusoe", ( - f"{model} should have crusoe as provider" - ) - assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + for model in crusoe_models: + model_info = get_model_info(model) + assert model_info is not None, f"Model info not found for {model}" + assert model_info.get("litellm_provider") == "crusoe", ( + f"{model} should have crusoe as provider" + ) + assert model_info.get("mode") == "chat", f"{model} should be in chat mode" + finally: + litellm.model_cost = original_model_cost + if original_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env diff --git a/tests/test_litellm/llms/crusoe/__init__.py b/tests/test_litellm/llms/crusoe/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 485c00b357..70c538d915 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -91,18 +91,27 @@ def test_crusoe_model_list_populated(): """Test Crusoe models are present in model_prices_and_context_window.json""" import litellm - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") + original_model_cost = litellm.model_cost + original_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + try: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") - expected = [ - "crusoe/meta-llama/Llama-3.3-70B-Instruct", - "crusoe/deepseek-ai/DeepSeek-R1-0528", - "crusoe/deepseek-ai/DeepSeek-V3-0324", - "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", - "crusoe/moonshotai/Kimi-K2-Thinking", - "crusoe/openai/gpt-oss-120b", - "crusoe/google/gemma-3-12b-it", - ] - for model in expected: - assert model in litellm.model_cost, f"{model} not found in model_cost" - assert litellm.model_cost[model].get("litellm_provider") == "crusoe" + expected = [ + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] + for model in expected: + assert model in litellm.model_cost, f"{model} not found in model_cost" + assert litellm.model_cost[model].get("litellm_provider") == "crusoe" + finally: + litellm.model_cost = original_model_cost + if original_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = original_env From 133512dcc6e0e93ba421c9b496a39da3aacaea94 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 20 Mar 2026 09:50:30 -0700 Subject: [PATCH 21/29] fix(crusoe): sync backup model cost map with main file The backup JSON was missing Crusoe model entries, causing test_crusoe_model_list_populated to fail with AssertionError. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...odel_prices_and_context_window_backup.json | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 13a45fd165..004568d761 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22061,6 +22061,95 @@ "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, + "crusoe/deepseek-ai/DeepSeek-R1-0528": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 7e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/deepseek-ai/DeepSeek-V3-0324": { + "input_cost_per_token": 1.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 163840, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/google/gemma-3-12b-it": { + "input_cost_per_token": 1e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/meta-llama/Llama-3.3-70B-Instruct": { + "input_cost_per_token": 2e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 2e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/moonshotai/Kimi-K2-Thinking": { + "input_cost_per_token": 2.5e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": false, + "supports_system_messages": true, + "supports_tool_choice": false + }, + "crusoe/openai/gpt-oss-120b": { + "input_cost_per_token": 8e-07, + "litellm_provider": "crusoe", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 8e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": { + "input_cost_per_token": 3e-06, + "litellm_provider": "crusoe", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", From 51f8e5a57bdedc7e3801a161e7c5267f1d6c32e7 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 20 Mar 2026 09:53:00 -0700 Subject: [PATCH 22/29] feat(crusoe): add supports_reasoning flag for DeepSeek-R1 and Kimi-K2-Thinking These are reasoning/thinking models but were missing the flag, causing litellm.supports_reasoning() to return False and reasoning-token handling to not activate. Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/model_prices_and_context_window_backup.json | 2 ++ model_prices_and_context_window.json | 2 ++ 2 files changed, 4 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 004568d761..0b82ed8160 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22070,6 +22070,7 @@ "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": false }, @@ -22121,6 +22122,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": false }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8a5a455e6b..a39b0c38b4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22118,6 +22118,7 @@ "mode": "chat", "output_cost_per_token": 7e-06, "supports_function_calling": false, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": false }, @@ -22169,6 +22170,7 @@ "mode": "chat", "output_cost_per_token": 2.5e-06, "supports_function_calling": false, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": false }, From 2805572e9b985fca6a31b380972a0d677d50f5ca Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 20 Mar 2026 10:00:57 -0700 Subject: [PATCH 23/29] =?UTF-8?q?fix(crusoe):=20add=20param=5Fmappings=20f?= =?UTF-8?q?or=20max=5Fcompletion=5Ftokens=20=E2=86=92=20max=5Ftokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crusoe's vLLM-based endpoint accepts max_tokens, not max_completion_tokens. Without this mapping, callers using the OpenAI-standard param would get errors. --- litellm/llms/openai_like/providers.json | 5 ++++- tests/test_litellm/llms/crusoe/test_crusoe.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index acbbd8b612..b5e5aa4ea2 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -110,6 +110,9 @@ "crusoe": { "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1", "api_key_env": "CRUSOE_API_KEY", - "api_base_env": "CRUSOE_API_BASE" + "api_base_env": "CRUSOE_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 70c538d915..877d13451b 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -78,6 +78,24 @@ def test_crusoe_supported_params(): assert "stream" in params +def test_crusoe_param_mapping_max_completion_tokens(): + """Test max_completion_tokens is mapped to max_tokens for Crusoe""" + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + config = create_config_class(JSONProviderRegistry.get("crusoe"))() + optional_params = config.map_openai_params( + non_default_params={"max_completion_tokens": 1024}, + optional_params={}, + model="meta-llama/Llama-3.3-70B-Instruct", + drop_params=False, + ) + + assert "max_tokens" in optional_params, "max_completion_tokens should be mapped to max_tokens" + assert optional_params["max_tokens"] == 1024 + assert "max_completion_tokens" not in optional_params + + def test_crusoe_provider_detection_by_prefix(): """Test crusoe/model prefix is correctly routed""" from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider From 6e1e6244cf125b6f9fbbe338e5bce1bdb15ca1a8 Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Fri, 20 Mar 2026 10:10:26 -0700 Subject: [PATCH 24/29] fix(crusoe): remove trailing slashes from API base URLs and fix list indentation Trailing slashes on custom API base examples cause double-slash in get_complete_url. Also fixes inconsistent list indentation in test_crusoe_models_configuration. --- docs/my-website/docs/providers/crusoe.md | 4 +-- tests/llm_translation/test_crusoe.py | 26 +++++++++---------- tests/test_litellm/llms/crusoe/test_crusoe.py | 8 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md index dbcd026498..3079c59312 100644 --- a/docs/my-website/docs/providers/crusoe.md +++ b/docs/my-website/docs/providers/crusoe.md @@ -153,14 +153,14 @@ import litellm from litellm import completion # Using environment variable -os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1/" +os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1" os.environ["CRUSOE_API_KEY"] = "" # your API key # Or pass directly response = completion( model="crusoe/meta-llama/Llama-3.3-70B-Instruct", messages=[{"content": "Hello!", "role": "user"}], - api_base="https://custom.crusoecloud.com/v1/", + api_base="https://custom.crusoecloud.com/v1", api_key="your-api-key" ) ``` diff --git a/tests/llm_translation/test_crusoe.py b/tests/llm_translation/test_crusoe.py index e570e314cd..56aa4e4cd4 100644 --- a/tests/llm_translation/test_crusoe.py +++ b/tests/llm_translation/test_crusoe.py @@ -39,11 +39,11 @@ def test_crusoe_get_openai_compatible_provider_info(): os.environ, { "CRUSOE_API_KEY": "test-key", - "CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1/", + "CRUSOE_API_BASE": "https://custom.crusoecloud.com/v1", }, ): api_base, api_key = config._get_openai_compatible_provider_info(None, None) - assert api_base == "https://custom.crusoecloud.com/v1/" + assert api_base == "https://custom.crusoecloud.com/v1" assert api_key == "test-key" # Test with explicit parameters (should override env vars) @@ -51,13 +51,13 @@ def test_crusoe_get_openai_compatible_provider_info(): os.environ, { "CRUSOE_API_KEY": "env-key", - "CRUSOE_API_BASE": "https://env.crusoecloud.com/v1/", + "CRUSOE_API_BASE": "https://env.crusoecloud.com/v1", }, ): api_base, api_key = config._get_openai_compatible_provider_info( - "https://param.crusoecloud.com/v1/", "param-key" + "https://param.crusoecloud.com/v1", "param-key" ) - assert api_base == "https://param.crusoecloud.com/v1/" + assert api_base == "https://param.crusoecloud.com/v1" assert api_key == "param-key" @@ -84,14 +84,14 @@ def test_crusoe_models_configuration(): litellm.model_cost = litellm.get_model_cost_map(url="") crusoe_models = [ - "crusoe/meta-llama/Llama-3.3-70B-Instruct", - "crusoe/deepseek-ai/DeepSeek-R1-0528", - "crusoe/deepseek-ai/DeepSeek-V3-0324", - "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", - "crusoe/moonshotai/Kimi-K2-Thinking", - "crusoe/openai/gpt-oss-120b", - "crusoe/google/gemma-3-12b-it", - ] + "crusoe/meta-llama/Llama-3.3-70B-Instruct", + "crusoe/deepseek-ai/DeepSeek-R1-0528", + "crusoe/deepseek-ai/DeepSeek-V3-0324", + "crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507", + "crusoe/moonshotai/Kimi-K2-Thinking", + "crusoe/openai/gpt-oss-120b", + "crusoe/google/gemma-3-12b-it", + ] for model in crusoe_models: model_info = get_model_info(model) diff --git a/tests/test_litellm/llms/crusoe/test_crusoe.py b/tests/test_litellm/llms/crusoe/test_crusoe.py index 877d13451b..0a05126919 100644 --- a/tests/test_litellm/llms/crusoe/test_crusoe.py +++ b/tests/test_litellm/llms/crusoe/test_crusoe.py @@ -39,11 +39,11 @@ def test_crusoe_dynamic_config_env_vars(): with patch.dict( os.environ, - {"CRUSOE_API_KEY": "test-key", "CRUSOE_API_BASE": "https://custom.crusoe.com/v1/"}, + {"CRUSOE_API_KEY": "test-key", "CRUSOE_API_BASE": "https://custom.crusoe.com/v1"}, ): api_base, api_key = config._get_openai_compatible_provider_info(None, None) - assert api_base == "https://custom.crusoe.com/v1/" + assert api_base == "https://custom.crusoe.com/v1" assert api_key == "test-key" @@ -56,10 +56,10 @@ def test_crusoe_dynamic_config_explicit_params(): with patch.dict(os.environ, {"CRUSOE_API_KEY": "env-key"}): api_base, api_key = config._get_openai_compatible_provider_info( - "https://override.crusoe.com/v1/", "override-key" + "https://override.crusoe.com/v1", "override-key" ) - assert api_base == "https://override.crusoe.com/v1/" + assert api_base == "https://override.crusoe.com/v1" assert api_key == "override-key" From e08b8ef7b6998f93e9ad3db0bd05746cdc95a17e Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Mon, 23 Mar 2026 12:53:10 -0700 Subject: [PATCH 25/29] fix(crusoe): split Custom API Base docs into two independent examples The previous example set CRUSOE_API_BASE via env var and also passed api_base= in the same call, making it look like both were required. They are independent alternatives. --- docs/my-website/docs/providers/crusoe.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md index 3079c59312..b28903c27c 100644 --- a/docs/my-website/docs/providers/crusoe.md +++ b/docs/my-website/docs/providers/crusoe.md @@ -147,21 +147,31 @@ model_list: ## Custom API Base -```python showLineNumbers title="Custom API Base" +**Option 1: Environment variable** + +```python showLineNumbers title="Custom API Base via env var" import os -import litellm from litellm import completion -# Using environment variable os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1" os.environ["CRUSOE_API_KEY"] = "" # your API key -# Or pass directly +response = completion( + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", + messages=[{"content": "Hello!", "role": "user"}], +) +``` + +**Option 2: Pass directly** + +```python showLineNumbers title="Custom API Base via parameter" +from litellm import completion + response = completion( model="crusoe/meta-llama/Llama-3.3-70B-Instruct", messages=[{"content": "Hello!", "role": "user"}], api_base="https://custom.crusoecloud.com/v1", - api_key="your-api-key" + api_key="your-api-key", ) ``` From f8ba2d750b1d05474f5b41943f6f3ca395976a1e Mon Sep 17 00:00:00 2001 From: Emmanuel Acheampong Date: Mon, 23 Mar 2026 13:43:25 -0700 Subject: [PATCH 26/29] fix(crusoe): fix streaming doc model typo and add supports_vision for Gemma 3 - Streaming example referenced Llama-3.1 instead of Llama-3.3 - Add supports_vision: true for gemma-3-12b-it in both JSON files, matching other providers (bedrock, novita) --- docs/my-website/docs/providers/crusoe.md | 2 +- litellm/model_prices_and_context_window_backup.json | 3 ++- model_prices_and_context_window.json | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/crusoe.md b/docs/my-website/docs/providers/crusoe.md index b28903c27c..aa737cbdcd 100644 --- a/docs/my-website/docs/providers/crusoe.md +++ b/docs/my-website/docs/providers/crusoe.md @@ -71,7 +71,7 @@ messages = [{"content": "Write a short story about AI", "role": "user"}] # Crusoe call with streaming response = completion( - model="crusoe/meta-llama/Llama-3.1-70B-Instruct", + model="crusoe/meta-llama/Llama-3.3-70B-Instruct", messages=messages, stream=True ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0b82ed8160..b690941e73 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22098,7 +22098,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "crusoe/meta-llama/Llama-3.3-70B-Instruct": { "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a39b0c38b4..70bec9c6ea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22146,7 +22146,8 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_system_messages": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "supports_vision": true }, "crusoe/meta-llama/Llama-3.3-70B-Instruct": { "input_cost_per_token": 2e-07, From 99d075d86351290d4f4f1480a6554302f3d2f223 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 20 Apr 2026 18:59:34 +0530 Subject: [PATCH 27/29] fix code qa --- provider_endpoints_support.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index ed49c14621..3fc7cd4318 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -635,6 +635,24 @@ "interactions": true } }, + "crusoe": { + "display_name": "Crusoe (`crusoe`)", + "url": "https://docs.litellm.ai/docs/providers/crusoe", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, "custom": { "display_name": "Custom (`custom`)", "url": "https://docs.litellm.ai/docs/providers/custom_llm_server", From 3eccededccfd28e7e9cc4934a8304383f60cd21b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 1 May 2026 17:33:50 +0530 Subject: [PATCH 28/29] Fix black --- litellm/_logging.py | 1 - litellm/cost_calculator.py | 4 +++- litellm/llms/azure/cost_calculation.py | 4 +++- litellm/llms/vertex_ai/gemini/transformation.py | 2 +- .../proxy/guardrails/guardrail_hooks/qohash/qohash.py | 9 +++++++-- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 8216777645..d072cc549d 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -24,7 +24,6 @@ _ENABLE_SECRET_REDACTION = ( _REDACTED = "REDACTED" - def _build_secret_patterns() -> re.Pattern: patterns: List[str] = [ # ── PEM private key / certificate blocks ── diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4d7b44bfbd..9b4dd80265 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -513,7 +513,9 @@ def cost_per_token( # noqa: PLR0915 return fireworks_ai_cost_per_token(model=model, usage=usage_block) elif custom_llm_provider == "azure": return azure_openai_cost_per_token( - model=model, usage=usage_block, response_time_ms=response_time_ms, + model=model, + usage=usage_block, + response_time_ms=response_time_ms, service_tier=service_tier, ) elif custom_llm_provider == "gemini": diff --git a/litellm/llms/azure/cost_calculation.py b/litellm/llms/azure/cost_calculation.py index 2e2e0c4965..2a20c55a6c 100644 --- a/litellm/llms/azure/cost_calculation.py +++ b/litellm/llms/azure/cost_calculation.py @@ -12,7 +12,9 @@ from litellm.utils import get_model_info def cost_per_token( - model: str, usage: Usage, response_time_ms: Optional[float] = 0.0, + model: str, + usage: Usage, + response_time_ms: Optional[float] = 0.0, service_tier: Optional[str] = None, ) -> Tuple[float, float]: """ diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 4d1b714db9..f756c7a7c5 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -225,7 +225,7 @@ def _process_gemini_media( # Gemini Files API references can be passed through as URI-only. file_data = cast(FileDataType, {"file_uri": image_url}) part = {"file_data": file_data} - return _apply_gemini_3_metadata( + return _apply_gemini_metadata( part, model, media_resolution_enum, video_metadata ) elif ( diff --git a/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py index 7de15192d6..a1bab6dbac 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qohash/qohash.py @@ -1,6 +1,7 @@ """ Qostodian Nexus (by Qohash) — LiteLLM guardrail integration. """ + import os from typing import TYPE_CHECKING, Literal, Optional, Type @@ -25,7 +26,9 @@ class QostodianNexus(GenericGuardrailAPI): api_base: Optional[str] = None, **kwargs, ): - api_base = api_base or os.environ.get("QOSTODIAN_NEXUS_API_BASE", "http://nexus:8800") + api_base = api_base or os.environ.get( + "QOSTODIAN_NEXUS_API_BASE", "http://nexus:8800" + ) kwargs["guardrail_name"] = kwargs.get("guardrail_name", GUARDRAIL_NAME) @@ -38,7 +41,9 @@ class QostodianNexus(GenericGuardrailAPI): ] existing = kwargs.get("extra_headers") or [] - kwargs["extra_headers"] = nexus_headers + [h for h in existing if h not in nexus_headers] + kwargs["extra_headers"] = nexus_headers + [ + h for h in existing if h not in nexus_headers + ] super().__init__( api_base=api_base, From 97cfb4f6fd62f6c70f9ba38a4b9ab5d4e145207b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 1 May 2026 17:24:21 +0000 Subject: [PATCH 29/29] Unify secret redaction patterns --- litellm/_logging.py | 69 +------------------ .../litellm_core_utils/secret_redaction.py | 19 ++++- tests/test_litellm/test_secret_redaction.py | 14 ++-- 3 files changed, 27 insertions(+), 75 deletions(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index d072cc549d..5ddafd6c6a 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,12 +1,12 @@ import ast import logging import os -import re import sys from datetime import datetime from logging import Formatter -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional +from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = ( os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true" ) -_REDACTED = "REDACTED" - - -def _build_secret_patterns() -> re.Pattern: - patterns: List[str] = [ - # ── PEM private key / certificate blocks ── - r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", - # ── GCP OAuth2 access tokens (ya29.*) ── - r"\bya29\.[A-Za-z0-9_.~+/-]+", - # ── Credential %s formatting (space separator, no key= prefix) ── - r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+", - # AWS access key IDs - r"(?:AKIA|ASIA)[0-9A-Z]{16}", - # AWS secrets / session tokens / access key IDs (key=value) - r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)" - r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}", - # Bearer tokens (OAuth, JWT, etc.) - r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", - # Basic auth headers - r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", - # OpenAI / Anthropic sk- prefixed keys - r"sk-[A-Za-z0-9\-_]{20,}", - # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr) - r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}", - # x-api-key / api-key header values (handles 'key': 'value' dict repr) - r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", - # Anthropic internal header keys - r"x-ak-[A-Za-z0-9\-_]{20,}", - # Google API keys - r"AIza[0-9A-Za-z\-_]{35}", - # Password / secret params (handles key=value and 'key': 'value') - # Word boundary prevents O(n^2) backtracking on long word-char runs. - r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" - r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", - # Database connection string credentials (scheme://user:pass@host) - r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", - # Databricks personal access tokens - r"dapi[0-9a-f]{32}", - # ── Key-name-based redaction ── - # Catches secrets inside dicts/config dumps by matching on the KEY name - # regardless of what the value looks like. - # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." - # private_key with PEM-aware value capture - r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", - r"(?:master_key|database_url|db_url|connection_string|" - r"signing_key|encryption_key|" - r"auth_token|access_token|refresh_token|" - r"slack_webhook_url|webhook_url|" - r"database_connection_string|" - r"huggingface_token|jwt_secret)" - r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", - # ── Raw JWTs (without Bearer prefix) ── - r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", - # ── Azure SAS tokens in URLs ── - r"[?&]sig=[A-Za-z0-9%+/=]+", - # ── Full JSON service-account blobs (single-line and multi-line) ── - r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', - ] - return re.compile("|".join(patterns), re.IGNORECASE) - - -_SECRET_RE = _build_secret_patterns() - def _redact_string(value: str) -> str: if not _ENABLE_SECRET_REDACTION: return value - return _SECRET_RE.sub(_REDACTED, value) + return redact_string(value) def redact_secrets(value: str) -> str: diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 4473c6539c..5c4e3e3dac 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -14,6 +14,12 @@ _REDACTED = "REDACTED" def _build_secret_patterns() -> "re.Pattern[str]": patterns: List[str] = [ + # PEM private key / certificate blocks + r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + # GCP OAuth2 access tokens (ya29.*) + r"\bya29\.[A-Za-z0-9_.~+/-]+", + # Credential %s formatting (space separator, no key= prefix) + r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+", # AWS access key IDs r"(?:AKIA|ASIA)[0-9A-Z]{16}", # AWS secrets / session tokens / access key IDs (key=value) @@ -37,7 +43,8 @@ def _build_secret_patterns() -> "re.Pattern[str]": # full "key=" fragment so the value is redacted regardless of format. r"(?<=[?&])key=[^\s&'\"]{8,}", # Password / secret params (handles key=value and 'key': 'value') - r"\w*(?:password|passwd|client_secret|secret_key|_secret)" + # Word boundary prevents O(n^2) backtracking on long word-char runs. + r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)" r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", # Database connection string credentials (scheme://user:pass@host) r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", @@ -47,13 +54,21 @@ def _build_secret_patterns() -> "re.Pattern[str]": # Catches secrets inside dicts/config dumps by matching on the KEY name # regardless of what the value looks like. # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." + # private_key with PEM-aware value capture + r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", r"(?:master_key|database_url|db_url|connection_string|" - r"private_key|signing_key|encryption_key|" + r"signing_key|encryption_key|" r"auth_token|access_token|refresh_token|" r"slack_webhook_url|webhook_url|" r"database_connection_string|" r"huggingface_token|jwt_secret)" r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""", + # Raw JWTs (without Bearer prefix) + r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", + # Azure SAS tokens in URLs + r"[?&]sig=[A-Za-z0-9%+/=]+", + # Full JSON service-account blobs (single-line and multi-line) + r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', ] return re.compile("|".join(patterns), re.IGNORECASE) diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index dbc339676e..8a0a2221c1 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -277,7 +277,7 @@ _SAMPLE_SA_JSON = ( def test_pem_private_key_redacted_in_json(): - result = _redact_string(_SAMPLE_SA_JSON) + result = redact_string(_SAMPLE_SA_JSON) assert "MIIEvQIBADA" not in result assert "-----BEGIN" not in result @@ -286,12 +286,12 @@ def test_pem_private_key_redacted_in_dict_repr(): import json sa = json.loads(_SAMPLE_SA_JSON) - result = _redact_string(str(sa)) + result = redact_string(str(sa)) assert "MIIEvQIBADA" not in result def test_service_account_blob_fully_redacted(): - result = _redact_string(f"Got={_SAMPLE_SA_JSON}") + result = redact_string(f"Got={_SAMPLE_SA_JSON}") assert "my-proj-123" not in result assert "sa@my-proj.iam.gserviceaccount.com" not in result assert "abc123def" not in result @@ -320,22 +320,22 @@ def test_vertex_traceback_redacts_pem(): "Unable to load vertex credentials from environment. " f"Got={_SAMPLE_SA_JSON}" ) - result = _redact_string(traceback_text) + result = redact_string(traceback_text) assert "MIIEvQIBADA" not in result assert "-----BEGIN" not in result def test_gcp_oauth_token_redacted(): - result = _redact_string("access token ya29.c.c0ASRK0GZvXlongtokenhere") + result = redact_string("access token ya29.c.c0ASRK0GZvXlongtokenhere") assert "ya29." not in result assert "REDACTED" in result def test_non_pem_private_key_value_redacted(): - result = _redact_string("'private_key': 'some-non-pem-secret-value'") + result = redact_string("'private_key': 'some-non-pem-secret-value'") assert "some-non-pem-secret" not in result def test_normal_vertex_log_not_redacted(): msg = "Vertex: Loading vertex credentials, is_file_path=True, current dir /app" - assert _redact_string(msg) == msg + assert redact_string(msg) == msg