mirror of
https://github.com/tiennm99/litellm.git
synced 2026-06-18 05:28:02 +00:00
Ishaan Jaff
36264d4764
* security add robots.txt settings to security * block web crawlers * test_enterprise_routes.py * docs proxy enterprise
28 lines
902 B
Python
28 lines
902 B
Python
from typing import Union, Optional
|
|
from litellm.secret_managers.main import str_to_bool
|
|
|
|
|
|
def _should_block_robots():
|
|
"""
|
|
Returns True if the robots.txt file should block web crawlers
|
|
|
|
Controlled by
|
|
|
|
```yaml
|
|
general_settings:
|
|
block_robots: true
|
|
```
|
|
"""
|
|
from litellm.proxy.proxy_server import general_settings, premium_user, CommonProxyErrors
|
|
_block_robots: Union[bool, str] = general_settings.get("block_robots", False)
|
|
block_robots: Optional[bool] = None
|
|
if isinstance(_block_robots, bool):
|
|
block_robots = _block_robots
|
|
elif isinstance(_block_robots, str):
|
|
block_robots = str_to_bool(_block_robots)
|
|
if block_robots is True:
|
|
if premium_user is not True:
|
|
raise ValueError(f"Blocking web crawlers is an enterprise feature. {CommonProxyErrors.not_premium_user.value}")
|
|
return True
|
|
return False
|