diff --git a/.circleci/config.yml b/.circleci/config.yml index ecbe022e56..2edbc32b1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1321,6 +1321,7 @@ jobs: # - run: python ./tests/documentation_tests/test_general_setting_keys.py - run: python ./tests/code_coverage_tests/check_licenses.py - run: python ./tests/code_coverage_tests/router_code_coverage.py + - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py - run: python ./tests/code_coverage_tests/code_qa_check_tests.py - run: python ./tests/code_coverage_tests/test_proxy_types_import.py - run: python ./tests/code_coverage_tests/callback_manager_test.py diff --git a/enterprise/enterprise_hooks/aporia_ai.py b/enterprise/enterprise_hooks/aporia_ai.py index 2b427bea5c..d2184e92f2 100644 --- a/enterprise/enterprise_hooks/aporia_ai.py +++ b/enterprise/enterprise_hooks/aporia_ai.py @@ -5,33 +5,32 @@ # +-------------------------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan -import sys import os +import sys sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -from typing import Optional, Literal, Any -import litellm +import json import sys -from litellm.proxy._types import UserAPIKeyAuth -from litellm.integrations.custom_guardrail import CustomGuardrail +from typing import Any, List, Literal, Optional + from fastapi import HTTPException + +import litellm from litellm._logging import verbose_proxy_logger -from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.logging_utils import ( convert_litellm_response_object_to_str, ) -from typing import List from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -import json +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata from litellm.types.guardrails import GuardrailEventHooks -litellm.set_verbose = True - GUARDRAIL_NAME = "aporia" diff --git a/enterprise/enterprise_hooks/openai_moderation.py b/enterprise/enterprise_hooks/openai_moderation.py index 1db932c853..ee8ac49509 100644 --- a/enterprise/enterprise_hooks/openai_moderation.py +++ b/enterprise/enterprise_hooks/openai_moderation.py @@ -5,21 +5,21 @@ # +-------------------------------------------------------------+ # Thank you users! We ❤️ you! - Krrish & Ishaan -import sys import os +import sys sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path -from typing import Literal -import litellm import sys -from litellm.proxy._types import UserAPIKeyAuth -from litellm.integrations.custom_logger import CustomLogger -from fastapi import HTTPException -from litellm._logging import verbose_proxy_logger +from typing import Literal -litellm.set_verbose = True +from fastapi import HTTPException + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import UserAPIKeyAuth class _ENTERPRISE_OpenAI_Moderation(CustomLogger): diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py index a2d77f51a4..a44af55d4b 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py @@ -25,8 +25,6 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import Choices, ModelResponse -litellm.set_verbose = True - class _ENTERPRISE_LlamaGuard(CustomLogger): # Class variables or attributes diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py index 59981154aa..1475a94303 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py @@ -19,8 +19,6 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.secret_managers.main import get_secret_str from litellm.utils import get_formatted_prompt -litellm.set_verbose = True - class _ENTERPRISE_LLMGuard(CustomLogger): # Class variables or attributes diff --git a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py index 8c61c0c104..85a0ab9ded 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aporia_ai/aporia_ai.py @@ -17,7 +17,6 @@ from typing import TYPE_CHECKING, Any, List, Literal, Optional, Type from fastapi import HTTPException -import litellm from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, @@ -33,8 +32,6 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks -litellm.set_verbose = True - GUARDRAIL_NAME = "aporia" if TYPE_CHECKING: diff --git a/tests/code_coverage_tests/test_ban_set_verbose.py b/tests/code_coverage_tests/test_ban_set_verbose.py new file mode 100644 index 0000000000..a003f666e7 --- /dev/null +++ b/tests/code_coverage_tests/test_ban_set_verbose.py @@ -0,0 +1,140 @@ +import ast +import os + + +def find_set_verbose_assignments(file_path): + """ + Finds all assignments of litellm.set_verbose = True in a given Python file. + Returns a list of tuples (line_number, assignment_text). + """ + try: + with open(file_path, "r", encoding="utf-8") as file: + content = file.read() + tree = ast.parse(content) + except (SyntaxError, UnicodeDecodeError) as e: + print(f"Warning: Error parsing file {file_path}: {e}") + return [] + + assignments = [] + content_lines = content.splitlines() + + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + # Check if this is an assignment to litellm.set_verbose + for target in node.targets: + if isinstance(target, ast.Attribute): + # Check if it's litellm.set_verbose + if (isinstance(target.value, ast.Name) and + target.value.id == "litellm" and + target.attr == "set_verbose"): + + # Check if the value being assigned is True + if (isinstance(node.value, ast.Constant) and + node.value.value is True): + line_num = node.lineno + line_text = content_lines[line_num - 1].strip() if line_num <= len(content_lines) else "" + assignments.append((line_num, line_text)) + elif (isinstance(node.value, ast.NameConstant) and + node.value.value is True): # For older Python versions + line_num = node.lineno + line_text = content_lines[line_num - 1].strip() if line_num <= len(content_lines) else "" + assignments.append((line_num, line_text)) + + return assignments + + +def scan_litellm_files(base_dir): + """ + Scans all Python files in the litellm directory for set_verbose assignments. + Returns a dictionary mapping file paths to lists of assignments. + """ + violations = {} + litellm_dirs = [ + "litellm", + "enterprise" + ] + + for litellm_dir in litellm_dirs: + dir_path = os.path.join(base_dir, litellm_dir) + if not os.path.exists(dir_path): + print(f"Warning: Directory {dir_path} does not exist.") + continue + + print(f"Scanning directory: {dir_path}") + for root, _, files in os.walk(dir_path): + for file in files: + if file.endswith(".py"): + file_path = os.path.join(root, file) + relative_path = os.path.relpath(file_path, base_dir) + + assignments = find_set_verbose_assignments(file_path) + if assignments: + violations[relative_path] = assignments + + return violations + + +def test_no_hardcoded_set_verbose(): + """ + Pytest-compatible test function that ensures no hardcoded litellm.set_verbose = True assignments exist. + """ + base_dir = "./" # Adjust path as needed for your setup + + violations = scan_litellm_files(base_dir) + + if violations: + violation_details = [] + total_violations = 0 + for file_path, assignments in violations.items(): + for line_num, line_text in assignments: + violation_details.append(f"{file_path}:{line_num} -> {line_text}") + total_violations += 1 + + error_msg = ( + f"Found {total_violations} prohibited litellm.set_verbose = True assignments:\n" + + "\n".join(violation_details) + + "\n\nREASON: litellm.set_verbose = True should not be hardcoded in production code. " + "Instead, use environment variables or configuration files to control verbosity." + ) + + raise AssertionError(error_msg) + + +def main(): + """ + Main function that scans for litellm.set_verbose = True assignments and fails if any are found. + """ + base_dir = "./" # Adjust path as needed for your setup + + print("Scanning for litellm.set_verbose = True assignments...") + violations = scan_litellm_files(base_dir) + + if violations: + print("\n❌ FOUND PROHIBITED litellm.set_verbose = True ASSIGNMENTS:") + print("=" * 60) + + total_violations = 0 + for file_path, assignments in violations.items(): + print(f"\nFile: {file_path}") + for line_num, line_text in assignments: + print(f" Line {line_num}: {line_text}") + total_violations += 1 + + print(f"\n📊 Total violations found: {total_violations}") + print("\n🚫 REASON: litellm.set_verbose = True should not be hardcoded in production code.") + print(" Instead, use environment variables or configuration files to control verbosity.") + print(" Example alternatives:") + print(" - Use LITELLM_LOG=DEBUG environment variable") + print(" - Use litellm.set_verbose = os.getenv('LITELLM_VERBOSE', 'false').lower() == 'true'") + print(" - Use configuration-based verbosity settings") + + raise Exception( + f"Found {total_violations} prohibited litellm.set_verbose = True assignments. " + "Remove these hardcoded verbosity settings and use configuration-based approaches instead." + ) + else: + print("✅ No prohibited litellm.set_verbose = True assignments found.") + + +if __name__ == "__main__": + main() \ No newline at end of file