mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-24 02:25:29 +00:00
Merge branch 'main' into litellm_pass_through_vtx_multi_modal
This commit is contained in:
Ishaan Jaff
@@ -282,7 +282,7 @@ jobs:
|
||||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install aiohttp
|
||||
pip install openai
|
||||
pip install "openai==1.40.0"
|
||||
python -m pip install --upgrade pip
|
||||
pip install "pydantic==2.7.1"
|
||||
pip install "pytest==7.3.1"
|
||||
|
||||
@@ -13,10 +13,11 @@ spec:
|
||||
{{- include "litellm.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
{{- with .Values.podAnnotations }}
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "litellm.labels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
|
||||
@@ -161,8 +161,7 @@ random_number = random.randint(
|
||||
print("testing semantic caching")
|
||||
litellm.cache = Cache(
|
||||
type="qdrant-semantic",
|
||||
qdrant_host_type="cloud", # can be either 'cloud' or 'local'
|
||||
qdrant_url=os.environ["QDRANT_URL"],
|
||||
qdrant_api_base=os.environ["QDRANT_API_BASE"],
|
||||
qdrant_api_key=os.environ["QDRANT_API_KEY"],
|
||||
qdrant_collection_name="your_collection_name", # any name of your collection
|
||||
similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
|
||||
@@ -491,12 +490,11 @@ def __init__(
|
||||
disk_cache_dir=None,
|
||||
|
||||
# qdrant cache params
|
||||
qdrant_url: Optional[str] = None,
|
||||
qdrant_api_base: Optional[str] = None,
|
||||
qdrant_api_key: Optional[str] = None,
|
||||
qdrant_collection_name: Optional[str] = None,
|
||||
qdrant_quantization_config: Optional[str] = None,
|
||||
qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
|
||||
qdrant_host_type: Optional[Literal["local","cloud"]] = "local",
|
||||
|
||||
**kwargs
|
||||
):
|
||||
|
||||
@@ -81,6 +81,7 @@ Works for:
|
||||
```python
|
||||
import os
|
||||
from litellm import completion
|
||||
from pydantic import BaseModel
|
||||
|
||||
# add to env var
|
||||
os.environ["OPENAI_API_KEY"] = ""
|
||||
|
||||
@@ -8,6 +8,7 @@ liteLLM supports:
|
||||
|
||||
- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
|
||||
- [Langfuse](https://langfuse.com/docs)
|
||||
- [LangSmith](https://www.langchain.com/langsmith)
|
||||
- [Helicone](https://docs.helicone.ai/introduction)
|
||||
- [Traceloop](https://traceloop.com/docs)
|
||||
- [Lunary](https://lunary.ai/docs)
|
||||
|
||||
@@ -56,7 +56,7 @@ response = litellm.completion(
|
||||
```
|
||||
|
||||
## Advanced
|
||||
### Set Langsmith fields - Custom Projec, Run names, tags
|
||||
### Set Langsmith fields
|
||||
|
||||
```python
|
||||
import litellm
|
||||
@@ -75,9 +75,17 @@ response = litellm.completion(
|
||||
{"role": "user", "content": "Hi 👋 - i'm openai"}
|
||||
],
|
||||
metadata={
|
||||
"run_name": "litellmRUN", # langsmith run name
|
||||
"project_name": "litellm-completion", # langsmith project name
|
||||
"tags": ["model1", "prod-2"] # tags to log on langsmith
|
||||
"run_name": "litellmRUN", # langsmith run name
|
||||
"project_name": "litellm-completion", # langsmith project name
|
||||
"run_id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", # langsmith run id
|
||||
"parent_run_id": "f8faf8c1-9778-49a4-9004-628cdb0047e5", # langsmith run parent run id
|
||||
"trace_id": "df570c03-5a03-4cea-8df0-c162d05127ac", # langsmith run trace id
|
||||
"session_id": "1ffd059c-17ea-40a8-8aef-70fd0307db82", # langsmith run session id
|
||||
"tags": ["model1", "prod-2"], # langsmith run tags
|
||||
"metadata": { # langsmith run metadata
|
||||
"key1": "value1"
|
||||
},
|
||||
"dotted_order": "20240429T004912090000Z497f6eca-6276-4993-bfeb-53cbbbba6f08"
|
||||
}
|
||||
)
|
||||
print(response)
|
||||
|
||||
@@ -131,6 +131,56 @@ Expected Response
|
||||
}
|
||||
```
|
||||
|
||||
## Add Streaming Support
|
||||
|
||||
Here's a simple example of returning unix epoch seconds for both completion + streaming use-cases.
|
||||
|
||||
s/o [@Eloy Lafuente](https://github.com/stronk7) for this code example.
|
||||
|
||||
```python
|
||||
import time
|
||||
from typing import Iterator, AsyncIterator
|
||||
from litellm.types.utils import GenericStreamingChunk, ModelResponse
|
||||
from litellm import CustomLLM, completion, acompletion
|
||||
|
||||
class UnixTimeLLM(CustomLLM):
|
||||
def completion(self, *args, **kwargs) -> ModelResponse:
|
||||
return completion(
|
||||
model="test/unixtime",
|
||||
mock_response=str(int(time.time())),
|
||||
) # type: ignore
|
||||
|
||||
async def acompletion(self, *args, **kwargs) -> ModelResponse:
|
||||
return await acompletion(
|
||||
model="test/unixtime",
|
||||
mock_response=str(int(time.time())),
|
||||
) # type: ignore
|
||||
|
||||
def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]:
|
||||
generic_streaming_chunk: GenericStreamingChunk = {
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"is_finished": True,
|
||||
"text": str(int(time.time())),
|
||||
"tool_use": None,
|
||||
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
return generic_streaming_chunk # type: ignore
|
||||
|
||||
async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]:
|
||||
generic_streaming_chunk: GenericStreamingChunk = {
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"is_finished": True,
|
||||
"text": str(int(time.time())),
|
||||
"tool_use": None,
|
||||
"usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0},
|
||||
}
|
||||
yield generic_streaming_chunk # type: ignore
|
||||
|
||||
unixtime = UnixTimeLLM()
|
||||
```
|
||||
|
||||
## Custom Handler Spec
|
||||
|
||||
```python
|
||||
|
||||
@@ -7,6 +7,7 @@ Cache LLM Responses
|
||||
LiteLLM supports:
|
||||
- In Memory Cache
|
||||
- Redis Cache
|
||||
- Qdrant Semantic Cache
|
||||
- Redis Semantic Cache
|
||||
- s3 Bucket Cache
|
||||
|
||||
@@ -103,6 +104,66 @@ $ litellm --config /path/to/config.yaml
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
|
||||
<TabItem value="qdrant-semantic" label="Qdrant Semantic cache">
|
||||
|
||||
Caching can be enabled by adding the `cache` key in the `config.yaml`
|
||||
|
||||
#### Step 1: Add `cache` to the config.yaml
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/fake
|
||||
api_key: fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
- model_name: openai-embedding
|
||||
litellm_params:
|
||||
model: openai/text-embedding-3-small
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
litellm_settings:
|
||||
set_verbose: True
|
||||
cache: True # set cache responses to True, litellm defaults to using a redis cache
|
||||
cache_params:
|
||||
type: qdrant-semantic
|
||||
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
|
||||
qdrant_collection_name: test_collection
|
||||
qdrant_quantization_config: binary
|
||||
similarity_threshold: 0.8 # similarity threshold for semantic cache
|
||||
```
|
||||
|
||||
#### Step 2: Add Qdrant Credentials to your .env
|
||||
|
||||
```shell
|
||||
QDRANT_API_KEY = "16rJUMBRx*************"
|
||||
QDRANT_API_BASE = "https://5392d382-45*********.cloud.qdrant.io"
|
||||
```
|
||||
|
||||
#### Step 3: Run proxy with config
|
||||
```shell
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
|
||||
#### Step 4. Test it
|
||||
|
||||
```shell
|
||||
curl -i http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "fake-openai-endpoint",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello"}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
**Expect to see `x-litellm-semantic-similarity` in the response headers when semantic caching is one**
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="s3" label="s3 cache">
|
||||
|
||||
#### Step 1: Add `cache` to the config.yaml
|
||||
@@ -182,6 +243,9 @@ REDIS_<redis-kwarg-name> = ""
|
||||
$ litellm --config /path/to/config.yaml
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
|
||||
|
||||
</Tabs>
|
||||
|
||||
|
||||
|
||||
@@ -728,6 +728,7 @@ general_settings:
|
||||
"disable_spend_logs": "boolean", # turn off writing each transaction to the db
|
||||
"disable_master_key_return": "boolean", # turn off returning master key on UI (checked on '/user/info' endpoint)
|
||||
"disable_reset_budget": "boolean", # turn off reset budget scheduled task
|
||||
"disable_adding_master_key_hash_to_db": "boolean", # turn off storing master key hash in db, for spend tracking
|
||||
"enable_jwt_auth": "boolean", # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
|
||||
"enforce_user_param": "boolean", # requires all openai endpoint requests to have a 'user' param
|
||||
"allowed_routes": "list", # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
|
||||
|
||||
@@ -61,6 +61,51 @@ litellm_settings:
|
||||
|
||||
Removes any field with `user_api_key_*` from metadata.
|
||||
|
||||
## What gets logged?
|
||||
|
||||
Found under `kwargs["standard_logging_payload"]`. This is a standard payload, logged for every response.
|
||||
|
||||
```python
|
||||
class StandardLoggingPayload(TypedDict):
|
||||
id: str
|
||||
call_type: str
|
||||
response_cost: float
|
||||
total_tokens: int
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
startTime: float
|
||||
endTime: float
|
||||
completionStartTime: float
|
||||
model_map_information: StandardLoggingModelInformation
|
||||
model: str
|
||||
model_id: Optional[str]
|
||||
model_group: Optional[str]
|
||||
api_base: str
|
||||
metadata: StandardLoggingMetadata
|
||||
cache_hit: Optional[bool]
|
||||
cache_key: Optional[str]
|
||||
saved_cache_cost: Optional[float]
|
||||
request_tags: list
|
||||
end_user: Optional[str]
|
||||
requester_ip_address: Optional[str]
|
||||
messages: Optional[Union[str, list, dict]]
|
||||
response: Optional[Union[str, list, dict]]
|
||||
model_parameters: dict
|
||||
hidden_params: StandardLoggingHiddenParams
|
||||
|
||||
class StandardLoggingHiddenParams(TypedDict):
|
||||
model_id: Optional[str]
|
||||
cache_key: Optional[str]
|
||||
api_base: Optional[str]
|
||||
response_cost: Optional[str]
|
||||
additional_headers: Optional[dict]
|
||||
|
||||
|
||||
class StandardLoggingModelInformation(TypedDict):
|
||||
model_map_key: str
|
||||
model_map_value: Optional[ModelInfo]
|
||||
```
|
||||
|
||||
## Logging Proxy Input/Output - Langfuse
|
||||
|
||||
We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment
|
||||
|
||||
@@ -333,4 +333,5 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
|
||||
```
|
||||
Key=... over available RPM=0. Model RPM=100, Active keys=None
|
||||
```
|
||||
```
|
||||
|
||||
|
||||
@@ -488,9 +488,34 @@ You can set:
|
||||
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="per-team" label="Per Team">
|
||||
|
||||
Use `/team/new` or `/team/update`, to persist rate limits across multiple keys for a team.
|
||||
|
||||
|
||||
```shell
|
||||
curl --location 'http://0.0.0.0:4000/team/new' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{"team_id": "my-prod-team", "max_parallel_requests": 10, "tpm_limit": 20, "rpm_limit": 4}'
|
||||
```
|
||||
|
||||
[**See Swagger**](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post)
|
||||
|
||||
**Expected Response**
|
||||
|
||||
```json
|
||||
{
|
||||
"key": "sk-sA7VDkyhlQ7m8Gt77Mbt3Q",
|
||||
"expires": "2024-01-19T01:21:12.816168",
|
||||
"team_id": "my-prod-team",
|
||||
}
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="per-user" label="Per Internal User">
|
||||
|
||||
Use `/user/new`, to persist rate limits across multiple keys.
|
||||
Use `/user/new` or `/user/update`, to persist rate limits across multiple keys for internal users.
|
||||
|
||||
|
||||
```shell
|
||||
@@ -653,6 +678,70 @@ curl --location 'http://localhost:4000/chat/completions' \
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Set default budget for ALL internal users
|
||||
|
||||
Use this to set a default budget for users who you give keys to.
|
||||
|
||||
This will apply when a user has [`user_role="internal_user"`](./self_serve.md#available-roles) (set this via `/user/new` or `/user/update`).
|
||||
|
||||
This will NOT apply if a key has a team_id (team budgets will apply then). [Tell us how we can improve this!](https://github.com/BerriAI/litellm/issues)
|
||||
|
||||
1. Define max budget in your config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: "gpt-3.5-turbo"
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
litellm_settings:
|
||||
max_internal_user_budget: 0 # amount in USD
|
||||
internal_user_budget_duration: "1mo" # reset every month
|
||||
```
|
||||
|
||||
2. Create key for user
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{}'
|
||||
```
|
||||
|
||||
Expected Response:
|
||||
|
||||
```bash
|
||||
{
|
||||
...
|
||||
"key": "sk-X53RdxnDhzamRwjKXR4IHg"
|
||||
}
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-X53RdxnDhzamRwjKXR4IHg' \
|
||||
-d '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
Expected Response:
|
||||
|
||||
```bash
|
||||
{
|
||||
"error": {
|
||||
"message": "ExceededBudget: User=<user_id> over budget. Spend=3.7e-05, Budget=0.0",
|
||||
"type": "budget_exceeded",
|
||||
"param": null,
|
||||
"code": "400"
|
||||
}
|
||||
}
|
||||
```
|
||||
## Grant Access to new model
|
||||
|
||||
Use model access groups to give users access to select models, and add new ones to it over time (e.g. mistral, llama-2, etc.).
|
||||
|
||||
@@ -74,6 +74,7 @@ const sidebars = {
|
||||
"proxy/alerting",
|
||||
"proxy/ui",
|
||||
"proxy/prometheus",
|
||||
"proxy/caching",
|
||||
"proxy/pass_through",
|
||||
"proxy/email",
|
||||
"proxy/multiple_admins",
|
||||
@@ -88,7 +89,6 @@ const sidebars = {
|
||||
"proxy/health",
|
||||
"proxy/debugging",
|
||||
"proxy/pii_masking",
|
||||
"proxy/caching",
|
||||
"proxy/call_hooks",
|
||||
"proxy/rules",
|
||||
"proxy/cli",
|
||||
|
||||
+36
-26
@@ -1223,7 +1223,7 @@ class RedisSemanticCache(BaseCache):
|
||||
class QdrantSemanticCache(BaseCache):
|
||||
def __init__(
|
||||
self,
|
||||
qdrant_url=None,
|
||||
qdrant_api_base=None,
|
||||
qdrant_api_key=None,
|
||||
collection_name=None,
|
||||
similarity_threshold=None,
|
||||
@@ -1251,18 +1251,31 @@ class QdrantSemanticCache(BaseCache):
|
||||
self.similarity_threshold = similarity_threshold
|
||||
self.embedding_model = embedding_model
|
||||
headers = {}
|
||||
if qdrant_url is None:
|
||||
qdrant_url = os.getenv("QDRANT_URL")
|
||||
if qdrant_api_key is None:
|
||||
qdrant_api_key = os.getenv("QDRANT_API_KEY")
|
||||
if qdrant_url is not None and qdrant_api_key is not None:
|
||||
headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"}
|
||||
else:
|
||||
raise Exception("Qdrant url and api_key must be")
|
||||
|
||||
self.qdrant_url = qdrant_url
|
||||
# check if defined as os.environ/ variable
|
||||
if qdrant_api_base:
|
||||
if isinstance(qdrant_api_base, str) and qdrant_api_base.startswith(
|
||||
"os.environ/"
|
||||
):
|
||||
qdrant_api_base = litellm.get_secret(qdrant_api_base)
|
||||
if qdrant_api_key:
|
||||
if isinstance(qdrant_api_key, str) and qdrant_api_key.startswith(
|
||||
"os.environ/"
|
||||
):
|
||||
qdrant_api_key = litellm.get_secret(qdrant_api_key)
|
||||
|
||||
qdrant_api_base = (
|
||||
qdrant_api_base or os.getenv("QDRANT_URL") or os.getenv("QDRANT_API_BASE")
|
||||
)
|
||||
qdrant_api_key = qdrant_api_key or os.getenv("QDRANT_API_KEY")
|
||||
headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"}
|
||||
|
||||
if qdrant_api_key is None or qdrant_api_base is None:
|
||||
raise ValueError("Qdrant url and api_key must be")
|
||||
|
||||
self.qdrant_api_base = qdrant_api_base
|
||||
self.qdrant_api_key = qdrant_api_key
|
||||
print_verbose(f"qdrant semantic-cache qdrant_url: {self.qdrant_url}")
|
||||
print_verbose(f"qdrant semantic-cache qdrant_api_base: {self.qdrant_api_base}")
|
||||
|
||||
self.headers = headers
|
||||
|
||||
@@ -1274,7 +1287,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
"Quantization config is not provided. Default binary quantization will be used."
|
||||
)
|
||||
collection_exists = self.sync_client.get(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}/exists",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/exists",
|
||||
headers=self.headers,
|
||||
)
|
||||
if collection_exists.status_code != 200:
|
||||
@@ -1284,7 +1297,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
|
||||
if collection_exists.json()["result"]["exists"]:
|
||||
collection_details = self.sync_client.get(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
|
||||
headers=self.headers,
|
||||
)
|
||||
self.collection_info = collection_details.json()
|
||||
@@ -1312,7 +1325,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
)
|
||||
|
||||
new_collection_status = self.sync_client.put(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
|
||||
json={
|
||||
"vectors": {"size": 1536, "distance": "Cosine"},
|
||||
"quantization_config": quantization_params,
|
||||
@@ -1321,7 +1334,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
)
|
||||
if new_collection_status.json()["result"]:
|
||||
collection_details = self.sync_client.get(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}",
|
||||
headers=self.headers,
|
||||
)
|
||||
self.collection_info = collection_details.json()
|
||||
@@ -1378,7 +1391,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
]
|
||||
}
|
||||
keys = self.sync_client.put(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}/points",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points",
|
||||
headers=self.headers,
|
||||
json=data,
|
||||
)
|
||||
@@ -1417,7 +1430,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
}
|
||||
|
||||
search_response = self.sync_client.post(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
|
||||
headers=self.headers,
|
||||
json=data,
|
||||
)
|
||||
@@ -1506,7 +1519,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
}
|
||||
|
||||
keys = await self.async_client.put(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}/points",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points",
|
||||
headers=self.headers,
|
||||
json=data,
|
||||
)
|
||||
@@ -1564,7 +1577,7 @@ class QdrantSemanticCache(BaseCache):
|
||||
}
|
||||
|
||||
search_response = await self.async_client.post(
|
||||
url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search",
|
||||
url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
|
||||
headers=self.headers,
|
||||
json=data,
|
||||
)
|
||||
@@ -2111,12 +2124,11 @@ class Cache:
|
||||
redis_semantic_cache_embedding_model="text-embedding-ada-002",
|
||||
redis_flush_size=None,
|
||||
disk_cache_dir=None,
|
||||
qdrant_url: Optional[str] = None,
|
||||
qdrant_api_base: Optional[str] = None,
|
||||
qdrant_api_key: Optional[str] = None,
|
||||
qdrant_collection_name: Optional[str] = None,
|
||||
qdrant_quantization_config: Optional[str] = None,
|
||||
qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
|
||||
qdrant_host_type: Optional[Literal["local", "cloud"]] = "local",
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@@ -2127,9 +2139,8 @@ class Cache:
|
||||
host (str, optional): The host address for the Redis cache. Required if type is "redis".
|
||||
port (int, optional): The port number for the Redis cache. Required if type is "redis".
|
||||
password (str, optional): The password for the Redis cache. Required if type is "redis".
|
||||
qdrant_url (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic".
|
||||
qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster. Required if qdrant_host_type is "cloud" and optional if qdrant_host_type is "local".
|
||||
qdrant_host_type (str, optional): Can be either "local" or "cloud". Should be "local" when you are running a local qdrant cluster or "cloud" when you are using a qdrant cloud cluster.
|
||||
qdrant_api_base (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic".
|
||||
qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster.
|
||||
qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
|
||||
similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
|
||||
|
||||
@@ -2158,13 +2169,12 @@ class Cache:
|
||||
)
|
||||
elif type == "qdrant-semantic":
|
||||
self.cache = QdrantSemanticCache(
|
||||
qdrant_url=qdrant_url,
|
||||
qdrant_api_base=qdrant_api_base,
|
||||
qdrant_api_key=qdrant_api_key,
|
||||
collection_name=qdrant_collection_name,
|
||||
similarity_threshold=similarity_threshold,
|
||||
quantization_config=qdrant_quantization_config,
|
||||
embedding_model=qdrant_semantic_cache_embedding_model,
|
||||
host_type=qdrant_host_type,
|
||||
)
|
||||
elif type == "local":
|
||||
self.cache = InMemoryCache()
|
||||
|
||||
@@ -98,6 +98,10 @@ class LangsmithLogger(CustomLogger):
|
||||
project_name = metadata.get("project_name", self.langsmith_project)
|
||||
run_name = metadata.get("run_name", self.langsmith_default_run_name)
|
||||
run_id = metadata.get("id", None)
|
||||
parent_run_id = metadata.get("parent_run_id", None)
|
||||
trace_id = metadata.get("trace_id", None)
|
||||
session_id = metadata.get("session_id", None)
|
||||
dotted_order = metadata.get("dotted_order", None)
|
||||
tags = metadata.get("tags", []) or []
|
||||
verbose_logger.debug(
|
||||
f"Langsmith Logging - project_name: {project_name}, run_name {run_name}"
|
||||
@@ -149,6 +153,18 @@ class LangsmithLogger(CustomLogger):
|
||||
if run_id:
|
||||
data["id"] = run_id
|
||||
|
||||
if parent_run_id:
|
||||
data["parent_run_id"] = parent_run_id
|
||||
|
||||
if trace_id:
|
||||
data["trace_id"] = trace_id
|
||||
|
||||
if session_id:
|
||||
data["session_id"] = session_id
|
||||
|
||||
if dotted_order:
|
||||
data["dotted_order"] = dotted_order
|
||||
|
||||
verbose_logger.debug("Langsmith Logging data on langsmith: %s", data)
|
||||
|
||||
return data
|
||||
|
||||
@@ -210,7 +210,7 @@ class Logging:
|
||||
self.optional_params = optional_params
|
||||
self.model = model
|
||||
self.user = user
|
||||
self.litellm_params = litellm_params
|
||||
self.litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
|
||||
self.logger_fn = litellm_params.get("logger_fn", None)
|
||||
verbose_logger.debug(f"self.optional_params: {self.optional_params}")
|
||||
|
||||
@@ -524,6 +524,7 @@ class Logging:
|
||||
TextCompletionResponse,
|
||||
HttpxBinaryResponseContent,
|
||||
],
|
||||
cache_hit: Optional[bool] = None,
|
||||
):
|
||||
"""
|
||||
Calculate response cost using result + logging object variables.
|
||||
@@ -535,10 +536,13 @@ class Logging:
|
||||
litellm_params=self.litellm_params
|
||||
)
|
||||
|
||||
if cache_hit is None:
|
||||
cache_hit = self.model_call_details.get("cache_hit", False)
|
||||
|
||||
response_cost = litellm.response_cost_calculator(
|
||||
response_object=result,
|
||||
model=self.model,
|
||||
cache_hit=self.model_call_details.get("cache_hit", False),
|
||||
cache_hit=cache_hit,
|
||||
custom_llm_provider=self.model_call_details.get(
|
||||
"custom_llm_provider", None
|
||||
),
|
||||
@@ -630,6 +634,7 @@ class Logging:
|
||||
init_response_obj=result,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
logging_obj=self,
|
||||
)
|
||||
)
|
||||
return start_time, end_time, result
|
||||
@@ -2181,6 +2186,7 @@ def get_standard_logging_object_payload(
|
||||
init_response_obj: Any,
|
||||
start_time: dt_object,
|
||||
end_time: dt_object,
|
||||
logging_obj: Logging,
|
||||
) -> Optional[StandardLoggingPayload]:
|
||||
try:
|
||||
if kwargs is None:
|
||||
@@ -2277,11 +2283,17 @@ def get_standard_logging_object_payload(
|
||||
cache_key = litellm.cache.get_cache_key(**kwargs)
|
||||
else:
|
||||
cache_key = None
|
||||
|
||||
saved_cache_cost: Optional[float] = None
|
||||
if cache_hit is True:
|
||||
import time
|
||||
|
||||
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
|
||||
|
||||
saved_cache_cost = logging_obj._response_cost_calculator(
|
||||
result=init_response_obj, cache_hit=False
|
||||
)
|
||||
|
||||
## Get model cost information ##
|
||||
base_model = _get_base_model_from_metadata(model_call_details=kwargs)
|
||||
custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
|
||||
@@ -2318,6 +2330,7 @@ def get_standard_logging_object_payload(
|
||||
id=str(id),
|
||||
call_type=call_type or "",
|
||||
cache_hit=cache_hit,
|
||||
saved_cache_cost=saved_cache_cost,
|
||||
startTime=start_time_float,
|
||||
endTime=end_time_float,
|
||||
completionStartTime=completion_start_time_float,
|
||||
@@ -2353,3 +2366,28 @@ def get_standard_logging_object_payload(
|
||||
"Error creating standard logging object - {}".format(str(e))
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
|
||||
if litellm_params is None:
|
||||
litellm_params = {}
|
||||
|
||||
metadata = litellm_params.get("metadata", {}) or {}
|
||||
|
||||
## check user_api_key_metadata for sensitive logging keys
|
||||
cleaned_user_api_key_metadata = {}
|
||||
if "user_api_key_metadata" in metadata and isinstance(
|
||||
metadata["user_api_key_metadata"], dict
|
||||
):
|
||||
for k, v in metadata["user_api_key_metadata"].items():
|
||||
if k == "logging": # prevent logging user logging keys
|
||||
cleaned_user_api_key_metadata[k] = (
|
||||
"scrubbed_by_litellm_for_sensitive_keys"
|
||||
)
|
||||
else:
|
||||
cleaned_user_api_key_metadata[k] = v
|
||||
|
||||
metadata["user_api_key_metadata"] = cleaned_user_api_key_metadata
|
||||
litellm_params["metadata"] = metadata
|
||||
|
||||
return litellm_params
|
||||
|
||||
@@ -84,6 +84,8 @@ class MistralConfig:
|
||||
|
||||
- `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'.
|
||||
|
||||
- `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array
|
||||
|
||||
- `random_seed` (integer or null): The seed to use for random sampling. If set, different calls will generate deterministic results.
|
||||
|
||||
- `safe_prompt` (boolean): Whether to inject a safety prompt before all conversations. API Default - 'false'.
|
||||
@@ -99,6 +101,7 @@ class MistralConfig:
|
||||
random_seed: Optional[int] = None
|
||||
safe_prompt: Optional[bool] = None
|
||||
response_format: Optional[dict] = None
|
||||
stop: Optional[Union[str, list]] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -110,6 +113,7 @@ class MistralConfig:
|
||||
random_seed: Optional[int] = None,
|
||||
safe_prompt: Optional[bool] = None,
|
||||
response_format: Optional[dict] = None,
|
||||
stop: Optional[Union[str, list]] = None
|
||||
) -> None:
|
||||
locals_ = locals().copy()
|
||||
for key, value in locals_.items():
|
||||
@@ -143,6 +147,7 @@ class MistralConfig:
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"seed",
|
||||
"stop",
|
||||
"response_format",
|
||||
]
|
||||
|
||||
@@ -166,6 +171,8 @@ class MistralConfig:
|
||||
optional_params["temperature"] = value
|
||||
if param == "top_p":
|
||||
optional_params["top_p"] = value
|
||||
if param == "stop":
|
||||
optional_params["stop"] = value
|
||||
if param == "tool_choice" and isinstance(value, str):
|
||||
optional_params["tool_choice"] = self._map_tool_choice(
|
||||
tool_choice=value
|
||||
|
||||
@@ -191,9 +191,11 @@ class GoogleAIStudioGeminiConfig: # key diff from VertexAI - 'frequency_penalty
|
||||
elif value["type"] == "text": # type: ignore
|
||||
optional_params["response_mime_type"] = "text/plain"
|
||||
if "response_schema" in value: # type: ignore
|
||||
optional_params["response_mime_type"] = "application/json"
|
||||
optional_params["response_schema"] = value["response_schema"] # type: ignore
|
||||
elif value["type"] == "json_schema": # type: ignore
|
||||
if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore
|
||||
optional_params["response_mime_type"] = "application/json"
|
||||
optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore
|
||||
if param == "tools" and isinstance(value, list):
|
||||
gtool_func_declarations = []
|
||||
@@ -403,9 +405,11 @@ class VertexGeminiConfig:
|
||||
elif value["type"] == "text":
|
||||
optional_params["response_mime_type"] = "text/plain"
|
||||
if "response_schema" in value:
|
||||
optional_params["response_mime_type"] = "application/json"
|
||||
optional_params["response_schema"] = value["response_schema"]
|
||||
elif value["type"] == "json_schema": # type: ignore
|
||||
if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore
|
||||
optional_params["response_mime_type"] = "application/json"
|
||||
optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore
|
||||
if param == "frequency_penalty":
|
||||
optional_params["frequency_penalty"] = value
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
model_list:
|
||||
- model_name: ollama/mistral
|
||||
- model_name: "*"
|
||||
litellm_params:
|
||||
model: ollama/mistral
|
||||
model: "*"
|
||||
|
||||
litellm_settings:
|
||||
success_callback: ["s3"]
|
||||
cache: true
|
||||
s3_callback_params:
|
||||
s3_bucket_name: mytestbucketlitellm # AWS Bucket Name for S3
|
||||
s3_region_name: us-west-2 # AWS Region Name for S3
|
||||
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
|
||||
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
|
||||
|
||||
@@ -21,6 +21,13 @@ else:
|
||||
Span = Any
|
||||
|
||||
|
||||
class LiteLLMTeamRoles(enum.Enum):
|
||||
# team admin
|
||||
TEAM_ADMIN = "admin"
|
||||
# team member
|
||||
TEAM_MEMBER = "user"
|
||||
|
||||
|
||||
class LitellmUserRoles(str, enum.Enum):
|
||||
"""
|
||||
Admin Roles:
|
||||
@@ -335,6 +342,11 @@ class LiteLLMRoutes(enum.Enum):
|
||||
+ sso_only_routes
|
||||
)
|
||||
|
||||
self_managed_routes: List = [
|
||||
"/team/member_add",
|
||||
"/team/member_delete",
|
||||
] # routes that manage their own allowed/disallowed logic
|
||||
|
||||
|
||||
# class LiteLLMAllowedRoutes(LiteLLMBase):
|
||||
# """
|
||||
@@ -1308,6 +1320,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
|
||||
soft_budget: Optional[float] = None
|
||||
team_model_aliases: Optional[Dict] = None
|
||||
team_member_spend: Optional[float] = None
|
||||
team_member: Optional[Member] = None
|
||||
team_metadata: Optional[Dict] = None
|
||||
|
||||
# End User Params
|
||||
|
||||
@@ -975,8 +975,6 @@ async def user_api_key_auth(
|
||||
if not _is_user_proxy_admin(user_obj=user_obj): # if non-admin
|
||||
if is_llm_api_route(route=route):
|
||||
pass
|
||||
elif is_llm_api_route(route=request["route"].name):
|
||||
pass
|
||||
elif (
|
||||
route in LiteLLMRoutes.info_routes.value
|
||||
): # check if user allowed to call an info route
|
||||
@@ -1046,11 +1044,16 @@ async def user_api_key_auth(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
|
||||
)
|
||||
|
||||
elif (
|
||||
_user_role == LitellmUserRoles.INTERNAL_USER.value
|
||||
and route in LiteLLMRoutes.internal_user_routes.value
|
||||
):
|
||||
pass
|
||||
elif (
|
||||
route in LiteLLMRoutes.self_managed_routes.value
|
||||
): # routes that manage their own allowed/disallowed logic
|
||||
pass
|
||||
else:
|
||||
user_role = "unknown"
|
||||
user_id = "unknown"
|
||||
|
||||
@@ -285,14 +285,18 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str,
|
||||
return headers
|
||||
|
||||
|
||||
def get_applied_guardrails_header(request_data: Dict) -> Optional[Dict]:
|
||||
def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]:
|
||||
_metadata = request_data.get("metadata", None) or {}
|
||||
headers = {}
|
||||
if "applied_guardrails" in _metadata:
|
||||
return {
|
||||
"x-litellm-applied-guardrails": ",".join(_metadata["applied_guardrails"]),
|
||||
}
|
||||
headers["x-litellm-applied-guardrails"] = ",".join(
|
||||
_metadata["applied_guardrails"]
|
||||
)
|
||||
|
||||
return None
|
||||
if "semantic-similarity" in _metadata:
|
||||
headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"])
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
def add_guardrail_to_applied_guardrails_header(
|
||||
|
||||
@@ -95,7 +95,9 @@ def convert_key_logging_metadata_to_callback(
|
||||
for var, value in data.callback_vars.items():
|
||||
if team_callback_settings_obj.callback_vars is None:
|
||||
team_callback_settings_obj.callback_vars = {}
|
||||
team_callback_settings_obj.callback_vars[var] = litellm.get_secret(value)
|
||||
team_callback_settings_obj.callback_vars[var] = (
|
||||
litellm.utils.get_secret(value, default_value=value) or value
|
||||
)
|
||||
|
||||
return team_callback_settings_obj
|
||||
|
||||
@@ -130,7 +132,6 @@ def _get_dynamic_logging_metadata(
|
||||
data=AddTeamCallback(**item),
|
||||
team_callback_settings_obj=callback_settings_obj,
|
||||
)
|
||||
|
||||
return callback_settings_obj
|
||||
|
||||
|
||||
|
||||
@@ -119,6 +119,7 @@ async def new_user(
|
||||
http_request=Request(
|
||||
scope={"type": "http", "path": "/user/new"},
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
if data.send_invite_email is True:
|
||||
|
||||
@@ -849,7 +849,7 @@ async def generate_key_helper_fn(
|
||||
}
|
||||
|
||||
if (
|
||||
litellm.get_secret("DISABLE_KEY_NAME", False) == True
|
||||
litellm.get_secret("DISABLE_KEY_NAME", False) is True
|
||||
): # allow user to disable storing abbreviated key name (shown in UI, to help figure out which key spent how much)
|
||||
pass
|
||||
else:
|
||||
|
||||
@@ -30,7 +30,7 @@ from litellm.proxy._types import (
|
||||
UpdateTeamRequest,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.auth.user_api_key_auth import _is_user_proxy_admin, user_api_key_auth
|
||||
from litellm.proxy.management_helpers.utils import (
|
||||
add_new_member,
|
||||
management_endpoint_wrapper,
|
||||
@@ -39,6 +39,16 @@ from litellm.proxy.management_helpers.utils import (
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _is_user_team_admin(
|
||||
user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable
|
||||
) -> bool:
|
||||
for member in team_obj.members_with_roles:
|
||||
if member.user_id is not None and member.user_id == user_api_key_dict.user_id:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
#### TEAM MANAGEMENT ####
|
||||
@router.post(
|
||||
"/team/new",
|
||||
@@ -417,6 +427,7 @@ async def team_member_add(
|
||||
|
||||
If user doesn't exist, new user row will also be added to User Table
|
||||
|
||||
Only proxy_admin or admin of team, allowed to access this endpoint.
|
||||
```
|
||||
|
||||
curl -X POST 'http://0.0.0.0:4000/team/member_add' \
|
||||
@@ -465,6 +476,24 @@ async def team_member_add(
|
||||
|
||||
complete_team_data = LiteLLM_TeamTable(**existing_team_row.model_dump())
|
||||
|
||||
## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
|
||||
|
||||
if (
|
||||
user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
|
||||
and not _is_user_team_admin(
|
||||
user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
|
||||
)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format(
|
||||
"/team/member_add",
|
||||
complete_team_data.team_id,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
if isinstance(data.member, Member):
|
||||
# add to team db
|
||||
new_member = data.member
|
||||
@@ -569,6 +598,23 @@ async def team_member_delete(
|
||||
)
|
||||
existing_team_row = LiteLLM_TeamTable(**_existing_team_row.model_dump())
|
||||
|
||||
## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
|
||||
|
||||
if (
|
||||
user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
|
||||
and not _is_user_team_admin(
|
||||
user_api_key_dict=user_api_key_dict, team_obj=existing_team_row
|
||||
)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format(
|
||||
"/team/member_delete", existing_team_row.team_id
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
## DELETE MEMBER FROM TEAM
|
||||
new_team_members: List[Member] = []
|
||||
for m in existing_team_row.members_with_roles:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
model_list:
|
||||
- model_name: multimodalembedding@001
|
||||
litellm_params:
|
||||
|
||||
model: vertex_ai/multimodalembedding@001
|
||||
vertex_project: "adroit-crow-413218"
|
||||
vertex_location: "us-central1"
|
||||
@@ -10,6 +11,3 @@ default_vertex_config:
|
||||
vertex_project: "adroit-crow-413218"
|
||||
vertex_location: "us-central1"
|
||||
vertex_credentials: adroit-crow-413218-a956eef1a2a8.json
|
||||
|
||||
litellm_settings:
|
||||
drop_params: True
|
||||
@@ -149,7 +149,7 @@ from litellm.proxy.common_utils.admin_ui_utils import (
|
||||
show_missing_vars_in_env,
|
||||
)
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
get_applied_guardrails_header,
|
||||
get_logging_caching_headers,
|
||||
get_remaining_tokens_and_requests_from_request_data,
|
||||
initialize_callbacks_on_proxy,
|
||||
)
|
||||
@@ -543,9 +543,9 @@ def get_custom_headers(
|
||||
)
|
||||
headers.update(remaining_tokens_header)
|
||||
|
||||
applied_guardrails = get_applied_guardrails_header(request_data)
|
||||
if applied_guardrails:
|
||||
headers.update(applied_guardrails)
|
||||
logging_caching_headers = get_logging_caching_headers(request_data)
|
||||
if logging_caching_headers:
|
||||
headers.update(logging_caching_headers)
|
||||
|
||||
try:
|
||||
return {
|
||||
@@ -2784,26 +2784,29 @@ async def startup_event():
|
||||
await custom_db_client.connect()
|
||||
|
||||
if prisma_client is not None and master_key is not None:
|
||||
# add master key to db
|
||||
if os.getenv("PROXY_ADMIN_ID", None) is not None:
|
||||
litellm_proxy_admin_name = os.getenv(
|
||||
"PROXY_ADMIN_ID", litellm_proxy_admin_name
|
||||
)
|
||||
asyncio.create_task(
|
||||
generate_key_helper_fn(
|
||||
request_type="user",
|
||||
duration=None,
|
||||
models=[],
|
||||
aliases={},
|
||||
config={},
|
||||
spend=0,
|
||||
token=master_key,
|
||||
user_id=litellm_proxy_admin_name,
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
query_type="update_data",
|
||||
update_key_values={"user_role": LitellmUserRoles.PROXY_ADMIN},
|
||||
if general_settings.get("disable_adding_master_key_hash_to_db") is True:
|
||||
verbose_proxy_logger.info("Skipping writing master key hash to db")
|
||||
else:
|
||||
# add master key to db
|
||||
asyncio.create_task(
|
||||
generate_key_helper_fn(
|
||||
request_type="user",
|
||||
duration=None,
|
||||
models=[],
|
||||
aliases={},
|
||||
config={},
|
||||
spend=0,
|
||||
token=master_key,
|
||||
user_id=litellm_proxy_admin_name,
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
query_type="update_data",
|
||||
update_key_values={"user_role": LitellmUserRoles.PROXY_ADMIN},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if prisma_client is not None and litellm.max_budget > 0:
|
||||
if litellm.budget_duration is None:
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import traceback
|
||||
from typing import Optional
|
||||
|
||||
@@ -8,12 +10,30 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
|
||||
def _is_master_key(api_key: str, _master_key: Optional[str]) -> bool:
|
||||
if _master_key is None:
|
||||
return False
|
||||
|
||||
## string comparison
|
||||
is_master_key = secrets.compare_digest(api_key, _master_key)
|
||||
if is_master_key:
|
||||
return True
|
||||
|
||||
## hash comparison
|
||||
is_master_key = secrets.compare_digest(api_key, hash_token(_master_key))
|
||||
if is_master_key:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_logging_payload(
|
||||
kwargs, response_obj, start_time, end_time, end_user_id: Optional[str]
|
||||
) -> SpendLogsPayload:
|
||||
from pydantic import Json
|
||||
|
||||
from litellm.proxy._types import LiteLLM_SpendLogs
|
||||
from litellm.proxy.proxy_server import general_settings, master_key
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
f"SpendTable: get_logging_payload - kwargs: {kwargs}\n\n"
|
||||
@@ -36,9 +56,15 @@ def get_logging_payload(
|
||||
usage = dict(usage)
|
||||
id = response_obj.get("id", kwargs.get("litellm_call_id"))
|
||||
api_key = metadata.get("user_api_key", "")
|
||||
if api_key is not None and isinstance(api_key, str) and api_key.startswith("sk-"):
|
||||
# hash the api_key
|
||||
api_key = hash_token(api_key)
|
||||
if api_key is not None and isinstance(api_key, str):
|
||||
if api_key.startswith("sk-"):
|
||||
# hash the api_key
|
||||
api_key = hash_token(api_key)
|
||||
if (
|
||||
_is_master_key(api_key=api_key, _master_key=master_key)
|
||||
and general_settings.get("disable_adding_master_key_hash_to_db") is True
|
||||
):
|
||||
api_key = "litellm_proxy_master_key" # use a known alias, if the user disabled storing master key in db
|
||||
|
||||
_model_id = metadata.get("model_info", {}).get("id", "")
|
||||
_model_group = metadata.get("model_group", "")
|
||||
|
||||
@@ -44,6 +44,7 @@ from litellm.proxy._types import (
|
||||
DynamoDBArgs,
|
||||
LiteLLM_VerificationTokenView,
|
||||
LitellmUserRoles,
|
||||
Member,
|
||||
ResetTeamBudgetRequest,
|
||||
SpendLogsMetadata,
|
||||
SpendLogsPayload,
|
||||
@@ -1395,6 +1396,7 @@ class PrismaClient:
|
||||
t.blocked AS team_blocked,
|
||||
t.team_alias AS team_alias,
|
||||
t.metadata AS team_metadata,
|
||||
t.members_with_roles AS team_members_with_roles,
|
||||
tm.spend AS team_member_spend,
|
||||
m.aliases as team_model_aliases
|
||||
FROM "LiteLLM_VerificationToken" AS v
|
||||
@@ -1412,6 +1414,33 @@ class PrismaClient:
|
||||
response["team_models"] = []
|
||||
if response["team_blocked"] is None:
|
||||
response["team_blocked"] = False
|
||||
|
||||
team_member: Optional[Member] = None
|
||||
if (
|
||||
response["team_members_with_roles"] is not None
|
||||
and response["user_id"] is not None
|
||||
):
|
||||
## find the team member corresponding to user id
|
||||
"""
|
||||
[
|
||||
{
|
||||
"role": "admin",
|
||||
"user_id": "default_user_id",
|
||||
"user_email": null
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"user_id": null,
|
||||
"user_email": "test@email.com"
|
||||
}
|
||||
]
|
||||
"""
|
||||
for tm in response["team_members_with_roles"]:
|
||||
if tm.get("user_id") is not None and response[
|
||||
"user_id"
|
||||
] == tm.get("user_id"):
|
||||
team_member = Member(**tm)
|
||||
response["team_member"] = team_member
|
||||
response = LiteLLM_VerificationTokenView(
|
||||
**response, last_refreshed_at=time.time()
|
||||
)
|
||||
|
||||
@@ -501,6 +501,8 @@ async def test_async_vertexai_streaming_response():
|
||||
assert len(complete_response) > 0
|
||||
except litellm.RateLimitError as e:
|
||||
pass
|
||||
except litellm.APIConnectionError:
|
||||
pass
|
||||
except litellm.Timeout as e:
|
||||
pass
|
||||
except litellm.InternalServerError as e:
|
||||
@@ -1558,6 +1560,16 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema(
|
||||
"response_schema"
|
||||
in mock_call.call_args.kwargs["json"]["generationConfig"]
|
||||
)
|
||||
assert (
|
||||
"response_mime_type"
|
||||
in mock_call.call_args.kwargs["json"]["generationConfig"]
|
||||
)
|
||||
assert (
|
||||
mock_call.call_args.kwargs["json"]["generationConfig"][
|
||||
"response_mime_type"
|
||||
]
|
||||
== "application/json"
|
||||
)
|
||||
else:
|
||||
assert (
|
||||
"response_schema"
|
||||
|
||||
@@ -1746,7 +1746,7 @@ async def test_qdrant_semantic_cache_acompletion():
|
||||
litellm.cache = Cache(
|
||||
type="qdrant-semantic",
|
||||
_host_type="cloud",
|
||||
qdrant_url=os.getenv("QDRANT_URL"),
|
||||
qdrant_api_base=os.getenv("QDRANT_URL"),
|
||||
qdrant_api_key=os.getenv("QDRANT_API_KEY"),
|
||||
qdrant_collection_name="test_collection",
|
||||
similarity_threshold=0.8,
|
||||
@@ -1794,8 +1794,7 @@ async def test_qdrant_semantic_cache_acompletion_stream():
|
||||
]
|
||||
litellm.cache = Cache(
|
||||
type="qdrant-semantic",
|
||||
qdrant_host_type="cloud",
|
||||
qdrant_url=os.getenv("QDRANT_URL"),
|
||||
qdrant_api_base=os.getenv("QDRANT_URL"),
|
||||
qdrant_api_key=os.getenv("QDRANT_API_KEY"),
|
||||
qdrant_collection_name="test_collection",
|
||||
similarity_threshold=0.8,
|
||||
|
||||
@@ -1252,3 +1252,45 @@ def test_standard_logging_payload(model, turn_off_message_logging):
|
||||
]
|
||||
if turn_off_message_logging:
|
||||
assert "redacted-by-litellm" == slobject["messages"][0]["content"]
|
||||
|
||||
|
||||
def test_standard_logging_payload_cache_hit():
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
# sync completion
|
||||
customHandler = CompletionCustomHandler()
|
||||
litellm.callbacks = [customHandler]
|
||||
|
||||
litellm.cache = Cache()
|
||||
|
||||
_ = litellm.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
caching=True,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
customHandler, "log_success_event", new=MagicMock()
|
||||
) as mock_client:
|
||||
_ = litellm.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
caching=True,
|
||||
)
|
||||
|
||||
time.sleep(2)
|
||||
mock_client.assert_called_once()
|
||||
|
||||
assert "standard_logging_object" in mock_client.call_args.kwargs["kwargs"]
|
||||
assert (
|
||||
mock_client.call_args.kwargs["kwargs"]["standard_logging_object"]
|
||||
is not None
|
||||
)
|
||||
|
||||
standard_logging_object: StandardLoggingPayload = mock_client.call_args.kwargs[
|
||||
"kwargs"
|
||||
]["standard_logging_object"]
|
||||
|
||||
assert standard_logging_object["cache_hit"] is True
|
||||
assert standard_logging_object["response_cost"] == 0
|
||||
assert standard_logging_object["saved_cache_cost"] > 0
|
||||
|
||||
@@ -909,7 +909,7 @@ async def test_create_team_member_add(prisma_client, new_member_method):
|
||||
|
||||
await team_member_add(
|
||||
data=team_member_add_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role="proxy_admin"),
|
||||
http_request=Request(
|
||||
scope={"type": "http", "path": "/user/new"},
|
||||
),
|
||||
@@ -930,6 +930,172 @@ async def test_create_team_member_add(prisma_client, new_member_method):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("team_member_role", ["admin", "user"])
|
||||
@pytest.mark.parametrize("team_route", ["/team/member_add", "/team/member_delete"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_team_member_add_team_admin_user_api_key_auth(
|
||||
prisma_client, team_member_role, team_route
|
||||
):
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import LiteLLM_TeamTableCachedObj, Member
|
||||
from litellm.proxy.proxy_server import (
|
||||
ProxyException,
|
||||
hash_token,
|
||||
user_api_key_auth,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
|
||||
setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
|
||||
setattr(litellm, "max_internal_user_budget", 10)
|
||||
setattr(litellm, "internal_user_budget_duration", "5m")
|
||||
await litellm.proxy.proxy_server.prisma_client.connect()
|
||||
user = f"ishaan {uuid.uuid4().hex}"
|
||||
_team_id = "litellm-test-client-id-new"
|
||||
user_key = "sk-12345678"
|
||||
|
||||
valid_token = UserAPIKeyAuth(
|
||||
team_id=_team_id,
|
||||
token=hash_token(user_key),
|
||||
team_member=Member(role=team_member_role, user_id=user),
|
||||
last_refreshed_at=time.time(),
|
||||
)
|
||||
user_api_key_cache.set_cache(key=hash_token(user_key), value=valid_token)
|
||||
|
||||
team_obj = LiteLLM_TeamTableCachedObj(
|
||||
team_id=_team_id,
|
||||
blocked=False,
|
||||
last_refreshed_at=time.time(),
|
||||
metadata={"guardrails": {"modify_guardrails": False}},
|
||||
)
|
||||
|
||||
user_api_key_cache.set_cache(key="team_id:{}".format(_team_id), value=team_obj)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache)
|
||||
|
||||
## TEST IF TEAM ADMIN ALLOWED TO CALL /MEMBER_ADD ENDPOINT
|
||||
import json
|
||||
|
||||
from starlette.datastructures import URL
|
||||
|
||||
request = Request(scope={"type": "http"})
|
||||
request._url = URL(url=team_route)
|
||||
|
||||
body = {}
|
||||
json_bytes = json.dumps(body).encode("utf-8")
|
||||
|
||||
request._body = json_bytes
|
||||
|
||||
## ALLOWED BY USER_API_KEY_AUTH
|
||||
await user_api_key_auth(request=request, api_key="Bearer " + user_key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("new_member_method", ["user_id", "user_email"])
|
||||
@pytest.mark.parametrize("user_role", ["admin", "user"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_team_member_add_team_admin(
|
||||
prisma_client, new_member_method, user_role
|
||||
):
|
||||
"""
|
||||
Relevant issue - https://github.com/BerriAI/litellm/issues/5300
|
||||
|
||||
Allow team admins to:
|
||||
- Add and remove team members
|
||||
- raise error if team member not an existing 'internal_user'
|
||||
"""
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import LiteLLM_TeamTableCachedObj, Member
|
||||
from litellm.proxy.proxy_server import (
|
||||
HTTPException,
|
||||
ProxyException,
|
||||
hash_token,
|
||||
user_api_key_auth,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
|
||||
setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
|
||||
setattr(litellm, "max_internal_user_budget", 10)
|
||||
setattr(litellm, "internal_user_budget_duration", "5m")
|
||||
await litellm.proxy.proxy_server.prisma_client.connect()
|
||||
user = f"ishaan {uuid.uuid4().hex}"
|
||||
_team_id = "litellm-test-client-id-new"
|
||||
user_key = "sk-12345678"
|
||||
|
||||
valid_token = UserAPIKeyAuth(
|
||||
team_id=_team_id,
|
||||
user_id=user,
|
||||
token=hash_token(user_key),
|
||||
last_refreshed_at=time.time(),
|
||||
)
|
||||
user_api_key_cache.set_cache(key=hash_token(user_key), value=valid_token)
|
||||
|
||||
team_obj = LiteLLM_TeamTableCachedObj(
|
||||
team_id=_team_id,
|
||||
blocked=False,
|
||||
last_refreshed_at=time.time(),
|
||||
members_with_roles=[Member(role=user_role, user_id=user)],
|
||||
metadata={"guardrails": {"modify_guardrails": False}},
|
||||
)
|
||||
|
||||
user_api_key_cache.set_cache(key="team_id:{}".format(_team_id), value=team_obj)
|
||||
|
||||
setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache)
|
||||
if new_member_method == "user_id":
|
||||
data = {
|
||||
"team_id": _team_id,
|
||||
"member": [{"role": "user", "user_id": user}],
|
||||
}
|
||||
elif new_member_method == "user_email":
|
||||
data = {
|
||||
"team_id": _team_id,
|
||||
"member": [{"role": "user", "user_email": user}],
|
||||
}
|
||||
team_member_add_request = TeamMemberAddRequest(**data)
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.proxy_server.prisma_client.db.litellm_usertable",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_litellm_usertable:
|
||||
mock_client = AsyncMock()
|
||||
mock_litellm_usertable.upsert = mock_client
|
||||
mock_litellm_usertable.find_many = AsyncMock(return_value=None)
|
||||
|
||||
try:
|
||||
await team_member_add(
|
||||
data=team_member_add_request,
|
||||
user_api_key_dict=valid_token,
|
||||
http_request=Request(
|
||||
scope={"type": "http", "path": "/user/new"},
|
||||
),
|
||||
)
|
||||
except HTTPException as e:
|
||||
if user_role == "user":
|
||||
assert e.status_code == 403
|
||||
else:
|
||||
raise e
|
||||
|
||||
mock_client.assert_called()
|
||||
|
||||
print(f"mock_client.call_args: {mock_client.call_args}")
|
||||
print("mock_client.call_args.kwargs: {}".format(mock_client.call_args.kwargs))
|
||||
|
||||
assert (
|
||||
mock_client.call_args.kwargs["data"]["create"]["max_budget"]
|
||||
== litellm.max_internal_user_budget
|
||||
)
|
||||
assert (
|
||||
mock_client.call_args.kwargs["data"]["create"]["budget_duration"]
|
||||
== litellm.internal_user_budget_duration
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_info_team_list(prisma_client):
|
||||
"""Assert user_info for admin calls team_list function"""
|
||||
@@ -1116,8 +1282,8 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client):
|
||||
"callback_name": "langfuse",
|
||||
"callback_type": "success",
|
||||
"callback_vars": {
|
||||
"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY",
|
||||
"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY",
|
||||
"langfuse_public_key": "my-mock-public-key",
|
||||
"langfuse_secret_key": "my-mock-secret-key",
|
||||
"langfuse_host": "https://us.cloud.langfuse.com",
|
||||
},
|
||||
}
|
||||
@@ -1165,7 +1331,9 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client):
|
||||
assert "success_callback" in new_data
|
||||
assert new_data["success_callback"] == ["langfuse"]
|
||||
assert "langfuse_public_key" in new_data
|
||||
assert new_data["langfuse_public_key"] == "my-mock-public-key"
|
||||
assert "langfuse_secret_key" in new_data
|
||||
assert new_data["langfuse_secret_key"] == "my-mock-secret-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -1218,6 +1218,7 @@ class StandardLoggingPayload(TypedDict):
|
||||
metadata: StandardLoggingMetadata
|
||||
cache_hit: Optional[bool]
|
||||
cache_key: Optional[str]
|
||||
saved_cache_cost: Optional[float]
|
||||
request_tags: list
|
||||
end_user: Optional[str]
|
||||
requester_ip_address: Optional[str]
|
||||
|
||||
+3
-1
@@ -8622,7 +8622,9 @@ def get_secret(
|
||||
return secret_value_as_bool
|
||||
else:
|
||||
return secret
|
||||
except:
|
||||
except Exception:
|
||||
if default_value is not None:
|
||||
return default_value
|
||||
return secret
|
||||
except Exception as e:
|
||||
if default_value is not None:
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.44.1"
|
||||
version = "1.44.2"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
@@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.44.1"
|
||||
version = "1.44.2"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user