diff --git a/.circleci/config.yml b/.circleci/config.yml index 854bb40f71..24d826f4f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -282,7 +282,7 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install openai + pip install "openai==1.40.0" python -m pip install --upgrade pip pip install "pydantic==2.7.1" pip install "pytest==7.3.1" diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 07e617581e..ab41e6cb53 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -13,10 +13,11 @@ spec: {{- include "litellm.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} labels: {{- include "litellm.labels" . | nindent 8 }} {{- with .Values.podLabels }} diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index c46f6d22cf..334adea3a3 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -161,8 +161,7 @@ random_number = random.randint( print("testing semantic caching") litellm.cache = Cache( type="qdrant-semantic", - qdrant_host_type="cloud", # can be either 'cloud' or 'local' - qdrant_url=os.environ["QDRANT_URL"], + qdrant_api_base=os.environ["QDRANT_API_BASE"], qdrant_api_key=os.environ["QDRANT_API_KEY"], qdrant_collection_name="your_collection_name", # any name of your collection similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity @@ -491,12 +490,11 @@ def __init__( disk_cache_dir=None, # qdrant cache params - qdrant_url: Optional[str] = None, + qdrant_api_base: Optional[str] = None, qdrant_api_key: Optional[str] = None, qdrant_collection_name: Optional[str] = None, qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model="text-embedding-ada-002", - qdrant_host_type: Optional[Literal["local","cloud"]] = "local", **kwargs ): diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index 6c325cd8d4..64cc22019d 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -81,6 +81,7 @@ Works for: ```python import os from litellm import completion +from pydantic import BaseModel # add to env var os.environ["OPENAI_API_KEY"] = "" diff --git a/docs/my-website/docs/observability/callbacks.md b/docs/my-website/docs/observability/callbacks.md index 0d54a89176..b959e8aae7 100644 --- a/docs/my-website/docs/observability/callbacks.md +++ b/docs/my-website/docs/observability/callbacks.md @@ -8,6 +8,7 @@ liteLLM supports: - [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback) - [Langfuse](https://langfuse.com/docs) +- [LangSmith](https://www.langchain.com/langsmith) - [Helicone](https://docs.helicone.ai/introduction) - [Traceloop](https://traceloop.com/docs) - [Lunary](https://lunary.ai/docs) diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index 88818e653b..e3eb171540 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -56,7 +56,7 @@ response = litellm.completion( ``` ## Advanced -### Set Langsmith fields - Custom Projec, Run names, tags +### Set Langsmith fields ```python import litellm @@ -75,9 +75,17 @@ response = litellm.completion( {"role": "user", "content": "Hi 👋 - i'm openai"} ], metadata={ - "run_name": "litellmRUN", # langsmith run name - "project_name": "litellm-completion", # langsmith project name - "tags": ["model1", "prod-2"] # tags to log on langsmith + "run_name": "litellmRUN", # langsmith run name + "project_name": "litellm-completion", # langsmith project name + "run_id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", # langsmith run id + "parent_run_id": "f8faf8c1-9778-49a4-9004-628cdb0047e5", # langsmith run parent run id + "trace_id": "df570c03-5a03-4cea-8df0-c162d05127ac", # langsmith run trace id + "session_id": "1ffd059c-17ea-40a8-8aef-70fd0307db82", # langsmith run session id + "tags": ["model1", "prod-2"], # langsmith run tags + "metadata": { # langsmith run metadata + "key1": "value1" + }, + "dotted_order": "20240429T004912090000Z497f6eca-6276-4993-bfeb-53cbbbba6f08" } ) print(response) diff --git a/docs/my-website/docs/providers/custom_llm_server.md b/docs/my-website/docs/providers/custom_llm_server.md index 400f45b7f0..7b2e1cde29 100644 --- a/docs/my-website/docs/providers/custom_llm_server.md +++ b/docs/my-website/docs/providers/custom_llm_server.md @@ -131,6 +131,56 @@ Expected Response } ``` +## Add Streaming Support + +Here's a simple example of returning unix epoch seconds for both completion + streaming use-cases. + +s/o [@Eloy Lafuente](https://github.com/stronk7) for this code example. + +```python +import time +from typing import Iterator, AsyncIterator +from litellm.types.utils import GenericStreamingChunk, ModelResponse +from litellm import CustomLLM, completion, acompletion + +class UnixTimeLLM(CustomLLM): + def completion(self, *args, **kwargs) -> ModelResponse: + return completion( + model="test/unixtime", + mock_response=str(int(time.time())), + ) # type: ignore + + async def acompletion(self, *args, **kwargs) -> ModelResponse: + return await acompletion( + model="test/unixtime", + mock_response=str(int(time.time())), + ) # type: ignore + + def streaming(self, *args, **kwargs) -> Iterator[GenericStreamingChunk]: + generic_streaming_chunk: GenericStreamingChunk = { + "finish_reason": "stop", + "index": 0, + "is_finished": True, + "text": str(int(time.time())), + "tool_use": None, + "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0}, + } + return generic_streaming_chunk # type: ignore + + async def astreaming(self, *args, **kwargs) -> AsyncIterator[GenericStreamingChunk]: + generic_streaming_chunk: GenericStreamingChunk = { + "finish_reason": "stop", + "index": 0, + "is_finished": True, + "text": str(int(time.time())), + "tool_use": None, + "usage": {"completion_tokens": 0, "prompt_tokens": 0, "total_tokens": 0}, + } + yield generic_streaming_chunk # type: ignore + +unixtime = UnixTimeLLM() +``` + ## Custom Handler Spec ```python diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index e1901a2e86..c2adca88a0 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -7,6 +7,7 @@ Cache LLM Responses LiteLLM supports: - In Memory Cache - Redis Cache +- Qdrant Semantic Cache - Redis Semantic Cache - s3 Bucket Cache @@ -103,6 +104,66 @@ $ litellm --config /path/to/config.yaml ``` + + + +Caching can be enabled by adding the `cache` key in the `config.yaml` + +#### Step 1: Add `cache` to the config.yaml +```yaml +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: openai-embedding + litellm_params: + model: openai/text-embedding-3-small + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + set_verbose: True + cache: True # set cache responses to True, litellm defaults to using a redis cache + cache_params: + type: qdrant-semantic + qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list + qdrant_collection_name: test_collection + qdrant_quantization_config: binary + similarity_threshold: 0.8 # similarity threshold for semantic cache +``` + +#### Step 2: Add Qdrant Credentials to your .env + +```shell +QDRANT_API_KEY = "16rJUMBRx*************" +QDRANT_API_BASE = "https://5392d382-45*********.cloud.qdrant.io" +``` + +#### Step 3: Run proxy with config +```shell +$ litellm --config /path/to/config.yaml +``` + + +#### Step 4. Test it + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "fake-openai-endpoint", + "messages": [ + {"role": "user", "content": "Hello"} + ] + }' +``` + +**Expect to see `x-litellm-semantic-similarity` in the response headers when semantic caching is one** + + + #### Step 1: Add `cache` to the config.yaml @@ -182,6 +243,9 @@ REDIS_ = "" $ litellm --config /path/to/config.yaml ``` + + + diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 1060db5e72..19c1f7902d 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -728,6 +728,7 @@ general_settings: "disable_spend_logs": "boolean", # turn off writing each transaction to the db "disable_master_key_return": "boolean", # turn off returning master key on UI (checked on '/user/info' endpoint) "disable_reset_budget": "boolean", # turn off reset budget scheduled task + "disable_adding_master_key_hash_to_db": "boolean", # turn off storing master key hash in db, for spend tracking "enable_jwt_auth": "boolean", # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims "enforce_user_param": "boolean", # requires all openai endpoint requests to have a 'user' param "allowed_routes": "list", # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 1be390ddd8..84ac8dfbe4 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -61,6 +61,51 @@ litellm_settings: Removes any field with `user_api_key_*` from metadata. +## What gets logged? + +Found under `kwargs["standard_logging_payload"]`. This is a standard payload, logged for every response. + +```python +class StandardLoggingPayload(TypedDict): + id: str + call_type: str + response_cost: float + total_tokens: int + prompt_tokens: int + completion_tokens: int + startTime: float + endTime: float + completionStartTime: float + model_map_information: StandardLoggingModelInformation + model: str + model_id: Optional[str] + model_group: Optional[str] + api_base: str + metadata: StandardLoggingMetadata + cache_hit: Optional[bool] + cache_key: Optional[str] + saved_cache_cost: Optional[float] + request_tags: list + end_user: Optional[str] + requester_ip_address: Optional[str] + messages: Optional[Union[str, list, dict]] + response: Optional[Union[str, list, dict]] + model_parameters: dict + hidden_params: StandardLoggingHiddenParams + +class StandardLoggingHiddenParams(TypedDict): + model_id: Optional[str] + cache_key: Optional[str] + api_base: Optional[str] + response_cost: Optional[str] + additional_headers: Optional[dict] + + +class StandardLoggingModelInformation(TypedDict): + model_map_key: str + model_map_value: Optional[ModelInfo] +``` + ## Logging Proxy Input/Output - Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md index d385264979..22b5242a0a 100644 --- a/docs/my-website/docs/proxy/team_budgets.md +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -333,4 +333,5 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ``` Key=... over available RPM=0. Model RPM=100, Active keys=None -``` \ No newline at end of file +``` + diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 522147708a..04f6e8c945 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -488,9 +488,34 @@ You can set: + + +Use `/team/new` or `/team/update`, to persist rate limits across multiple keys for a team. + + +```shell +curl --location 'http://0.0.0.0:4000/team/new' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{"team_id": "my-prod-team", "max_parallel_requests": 10, "tpm_limit": 20, "rpm_limit": 4}' +``` + +[**See Swagger**](https://litellm-api.up.railway.app/#/team%20management/new_team_team_new_post) + +**Expected Response** + +```json +{ + "key": "sk-sA7VDkyhlQ7m8Gt77Mbt3Q", + "expires": "2024-01-19T01:21:12.816168", + "team_id": "my-prod-team", +} +``` + + -Use `/user/new`, to persist rate limits across multiple keys. +Use `/user/new` or `/user/update`, to persist rate limits across multiple keys for internal users. ```shell @@ -653,6 +678,70 @@ curl --location 'http://localhost:4000/chat/completions' \ +## Set default budget for ALL internal users + +Use this to set a default budget for users who you give keys to. + +This will apply when a user has [`user_role="internal_user"`](./self_serve.md#available-roles) (set this via `/user/new` or `/user/update`). + +This will NOT apply if a key has a team_id (team budgets will apply then). [Tell us how we can improve this!](https://github.com/BerriAI/litellm/issues) + +1. Define max budget in your config.yaml + +```yaml +model_list: + - model_name: "gpt-3.5-turbo" + litellm_params: + model: gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + max_internal_user_budget: 0 # amount in USD + internal_user_budget_duration: "1mo" # reset every month +``` + +2. Create key for user + +```bash +curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{}' +``` + +Expected Response: + +```bash +{ + ... + "key": "sk-X53RdxnDhzamRwjKXR4IHg" +} +``` + +3. Test it! + +```bash +curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-X53RdxnDhzamRwjKXR4IHg' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hey, how's it going?"}] +}' +``` + +Expected Response: + +```bash +{ + "error": { + "message": "ExceededBudget: User= over budget. Spend=3.7e-05, Budget=0.0", + "type": "budget_exceeded", + "param": null, + "code": "400" + } +} +``` ## Grant Access to new model Use model access groups to give users access to select models, and add new ones to it over time (e.g. mistral, llama-2, etc.). diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8f4c33bead..ab94ed5b42 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -74,6 +74,7 @@ const sidebars = { "proxy/alerting", "proxy/ui", "proxy/prometheus", + "proxy/caching", "proxy/pass_through", "proxy/email", "proxy/multiple_admins", @@ -88,7 +89,6 @@ const sidebars = { "proxy/health", "proxy/debugging", "proxy/pii_masking", - "proxy/caching", "proxy/call_hooks", "proxy/rules", "proxy/cli", diff --git a/litellm/caching.py b/litellm/caching.py index ce224e610e..1c72160295 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1223,7 +1223,7 @@ class RedisSemanticCache(BaseCache): class QdrantSemanticCache(BaseCache): def __init__( self, - qdrant_url=None, + qdrant_api_base=None, qdrant_api_key=None, collection_name=None, similarity_threshold=None, @@ -1251,18 +1251,31 @@ class QdrantSemanticCache(BaseCache): self.similarity_threshold = similarity_threshold self.embedding_model = embedding_model headers = {} - if qdrant_url is None: - qdrant_url = os.getenv("QDRANT_URL") - if qdrant_api_key is None: - qdrant_api_key = os.getenv("QDRANT_API_KEY") - if qdrant_url is not None and qdrant_api_key is not None: - headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"} - else: - raise Exception("Qdrant url and api_key must be") - self.qdrant_url = qdrant_url + # check if defined as os.environ/ variable + if qdrant_api_base: + if isinstance(qdrant_api_base, str) and qdrant_api_base.startswith( + "os.environ/" + ): + qdrant_api_base = litellm.get_secret(qdrant_api_base) + if qdrant_api_key: + if isinstance(qdrant_api_key, str) and qdrant_api_key.startswith( + "os.environ/" + ): + qdrant_api_key = litellm.get_secret(qdrant_api_key) + + qdrant_api_base = ( + qdrant_api_base or os.getenv("QDRANT_URL") or os.getenv("QDRANT_API_BASE") + ) + qdrant_api_key = qdrant_api_key or os.getenv("QDRANT_API_KEY") + headers = {"api-key": qdrant_api_key, "Content-Type": "application/json"} + + if qdrant_api_key is None or qdrant_api_base is None: + raise ValueError("Qdrant url and api_key must be") + + self.qdrant_api_base = qdrant_api_base self.qdrant_api_key = qdrant_api_key - print_verbose(f"qdrant semantic-cache qdrant_url: {self.qdrant_url}") + print_verbose(f"qdrant semantic-cache qdrant_api_base: {self.qdrant_api_base}") self.headers = headers @@ -1274,7 +1287,7 @@ class QdrantSemanticCache(BaseCache): "Quantization config is not provided. Default binary quantization will be used." ) collection_exists = self.sync_client.get( - url=f"{self.qdrant_url}/collections/{self.collection_name}/exists", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/exists", headers=self.headers, ) if collection_exists.status_code != 200: @@ -1284,7 +1297,7 @@ class QdrantSemanticCache(BaseCache): if collection_exists.json()["result"]["exists"]: collection_details = self.sync_client.get( - url=f"{self.qdrant_url}/collections/{self.collection_name}", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}", headers=self.headers, ) self.collection_info = collection_details.json() @@ -1312,7 +1325,7 @@ class QdrantSemanticCache(BaseCache): ) new_collection_status = self.sync_client.put( - url=f"{self.qdrant_url}/collections/{self.collection_name}", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}", json={ "vectors": {"size": 1536, "distance": "Cosine"}, "quantization_config": quantization_params, @@ -1321,7 +1334,7 @@ class QdrantSemanticCache(BaseCache): ) if new_collection_status.json()["result"]: collection_details = self.sync_client.get( - url=f"{self.qdrant_url}/collections/{self.collection_name}", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}", headers=self.headers, ) self.collection_info = collection_details.json() @@ -1378,7 +1391,7 @@ class QdrantSemanticCache(BaseCache): ] } keys = self.sync_client.put( - url=f"{self.qdrant_url}/collections/{self.collection_name}/points", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points", headers=self.headers, json=data, ) @@ -1417,7 +1430,7 @@ class QdrantSemanticCache(BaseCache): } search_response = self.sync_client.post( - url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", headers=self.headers, json=data, ) @@ -1506,7 +1519,7 @@ class QdrantSemanticCache(BaseCache): } keys = await self.async_client.put( - url=f"{self.qdrant_url}/collections/{self.collection_name}/points", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points", headers=self.headers, json=data, ) @@ -1564,7 +1577,7 @@ class QdrantSemanticCache(BaseCache): } search_response = await self.async_client.post( - url=f"{self.qdrant_url}/collections/{self.collection_name}/points/search", + url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search", headers=self.headers, json=data, ) @@ -2111,12 +2124,11 @@ class Cache: redis_semantic_cache_embedding_model="text-embedding-ada-002", redis_flush_size=None, disk_cache_dir=None, - qdrant_url: Optional[str] = None, + qdrant_api_base: Optional[str] = None, qdrant_api_key: Optional[str] = None, qdrant_collection_name: Optional[str] = None, qdrant_quantization_config: Optional[str] = None, qdrant_semantic_cache_embedding_model="text-embedding-ada-002", - qdrant_host_type: Optional[Literal["local", "cloud"]] = "local", **kwargs, ): """ @@ -2127,9 +2139,8 @@ class Cache: host (str, optional): The host address for the Redis cache. Required if type is "redis". port (int, optional): The port number for the Redis cache. Required if type is "redis". password (str, optional): The password for the Redis cache. Required if type is "redis". - qdrant_url (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic". - qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster. Required if qdrant_host_type is "cloud" and optional if qdrant_host_type is "local". - qdrant_host_type (str, optional): Can be either "local" or "cloud". Should be "local" when you are running a local qdrant cluster or "cloud" when you are using a qdrant cloud cluster. + qdrant_api_base (str, optional): The url for your qdrant cluster. Required if type is "qdrant-semantic". + qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster. qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic". similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic". @@ -2158,13 +2169,12 @@ class Cache: ) elif type == "qdrant-semantic": self.cache = QdrantSemanticCache( - qdrant_url=qdrant_url, + qdrant_api_base=qdrant_api_base, qdrant_api_key=qdrant_api_key, collection_name=qdrant_collection_name, similarity_threshold=similarity_threshold, quantization_config=qdrant_quantization_config, embedding_model=qdrant_semantic_cache_embedding_model, - host_type=qdrant_host_type, ) elif type == "local": self.cache = InMemoryCache() diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 97242a2616..c14a6bacc9 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -98,6 +98,10 @@ class LangsmithLogger(CustomLogger): project_name = metadata.get("project_name", self.langsmith_project) run_name = metadata.get("run_name", self.langsmith_default_run_name) run_id = metadata.get("id", None) + parent_run_id = metadata.get("parent_run_id", None) + trace_id = metadata.get("trace_id", None) + session_id = metadata.get("session_id", None) + dotted_order = metadata.get("dotted_order", None) tags = metadata.get("tags", []) or [] verbose_logger.debug( f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" @@ -149,6 +153,18 @@ class LangsmithLogger(CustomLogger): if run_id: data["id"] = run_id + if parent_run_id: + data["parent_run_id"] = parent_run_id + + if trace_id: + data["trace_id"] = trace_id + + if session_id: + data["session_id"] = session_id + + if dotted_order: + data["dotted_order"] = dotted_order + verbose_logger.debug("Langsmith Logging data on langsmith: %s", data) return data diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7e7cb506fe..d59f985584 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -210,7 +210,7 @@ class Logging: self.optional_params = optional_params self.model = model self.user = user - self.litellm_params = litellm_params + self.litellm_params = scrub_sensitive_keys_in_metadata(litellm_params) self.logger_fn = litellm_params.get("logger_fn", None) verbose_logger.debug(f"self.optional_params: {self.optional_params}") @@ -524,6 +524,7 @@ class Logging: TextCompletionResponse, HttpxBinaryResponseContent, ], + cache_hit: Optional[bool] = None, ): """ Calculate response cost using result + logging object variables. @@ -535,10 +536,13 @@ class Logging: litellm_params=self.litellm_params ) + if cache_hit is None: + cache_hit = self.model_call_details.get("cache_hit", False) + response_cost = litellm.response_cost_calculator( response_object=result, model=self.model, - cache_hit=self.model_call_details.get("cache_hit", False), + cache_hit=cache_hit, custom_llm_provider=self.model_call_details.get( "custom_llm_provider", None ), @@ -630,6 +634,7 @@ class Logging: init_response_obj=result, start_time=start_time, end_time=end_time, + logging_obj=self, ) ) return start_time, end_time, result @@ -2181,6 +2186,7 @@ def get_standard_logging_object_payload( init_response_obj: Any, start_time: dt_object, end_time: dt_object, + logging_obj: Logging, ) -> Optional[StandardLoggingPayload]: try: if kwargs is None: @@ -2277,11 +2283,17 @@ def get_standard_logging_object_payload( cache_key = litellm.cache.get_cache_key(**kwargs) else: cache_key = None + + saved_cache_cost: Optional[float] = None if cache_hit is True: import time id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id + saved_cache_cost = logging_obj._response_cost_calculator( + result=init_response_obj, cache_hit=False + ) + ## Get model cost information ## base_model = _get_base_model_from_metadata(model_call_details=kwargs) custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params) @@ -2318,6 +2330,7 @@ def get_standard_logging_object_payload( id=str(id), call_type=call_type or "", cache_hit=cache_hit, + saved_cache_cost=saved_cache_cost, startTime=start_time_float, endTime=end_time_float, completionStartTime=completion_start_time_float, @@ -2353,3 +2366,28 @@ def get_standard_logging_object_payload( "Error creating standard logging object - {}".format(str(e)) ) return None + + +def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]): + if litellm_params is None: + litellm_params = {} + + metadata = litellm_params.get("metadata", {}) or {} + + ## check user_api_key_metadata for sensitive logging keys + cleaned_user_api_key_metadata = {} + if "user_api_key_metadata" in metadata and isinstance( + metadata["user_api_key_metadata"], dict + ): + for k, v in metadata["user_api_key_metadata"].items(): + if k == "logging": # prevent logging user logging keys + cleaned_user_api_key_metadata[k] = ( + "scrubbed_by_litellm_for_sensitive_keys" + ) + else: + cleaned_user_api_key_metadata[k] = v + + metadata["user_api_key_metadata"] = cleaned_user_api_key_metadata + litellm_params["metadata"] = metadata + + return litellm_params diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 3db001183d..ada5f4ca3d 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -84,6 +84,8 @@ class MistralConfig: - `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'. + - `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array + - `random_seed` (integer or null): The seed to use for random sampling. If set, different calls will generate deterministic results. - `safe_prompt` (boolean): Whether to inject a safety prompt before all conversations. API Default - 'false'. @@ -99,6 +101,7 @@ class MistralConfig: random_seed: Optional[int] = None safe_prompt: Optional[bool] = None response_format: Optional[dict] = None + stop: Optional[Union[str, list]] = None def __init__( self, @@ -110,6 +113,7 @@ class MistralConfig: random_seed: Optional[int] = None, safe_prompt: Optional[bool] = None, response_format: Optional[dict] = None, + stop: Optional[Union[str, list]] = None ) -> None: locals_ = locals().copy() for key, value in locals_.items(): @@ -143,6 +147,7 @@ class MistralConfig: "tools", "tool_choice", "seed", + "stop", "response_format", ] @@ -166,6 +171,8 @@ class MistralConfig: optional_params["temperature"] = value if param == "top_p": optional_params["top_p"] = value + if param == "stop": + optional_params["stop"] = value if param == "tool_choice" and isinstance(value, str): optional_params["tool_choice"] = self._map_tool_choice( tool_choice=value diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index c784bf2e89..a88925330a 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -191,9 +191,11 @@ class GoogleAIStudioGeminiConfig: # key diff from VertexAI - 'frequency_penalty elif value["type"] == "text": # type: ignore optional_params["response_mime_type"] = "text/plain" if "response_schema" in value: # type: ignore + optional_params["response_mime_type"] = "application/json" optional_params["response_schema"] = value["response_schema"] # type: ignore elif value["type"] == "json_schema": # type: ignore if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore + optional_params["response_mime_type"] = "application/json" optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore if param == "tools" and isinstance(value, list): gtool_func_declarations = [] @@ -403,9 +405,11 @@ class VertexGeminiConfig: elif value["type"] == "text": optional_params["response_mime_type"] = "text/plain" if "response_schema" in value: + optional_params["response_mime_type"] = "application/json" optional_params["response_schema"] = value["response_schema"] elif value["type"] == "json_schema": # type: ignore if "json_schema" in value and "schema" in value["json_schema"]: # type: ignore + optional_params["response_mime_type"] = "application/json" optional_params["response_schema"] = value["json_schema"]["schema"] # type: ignore if param == "frequency_penalty": optional_params["frequency_penalty"] = value diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index f72e372786..2c888a4f30 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,4 +1,13 @@ model_list: - - model_name: ollama/mistral + - model_name: "*" litellm_params: - model: ollama/mistral + model: "*" + +litellm_settings: + success_callback: ["s3"] + cache: true + s3_callback_params: + s3_bucket_name: mytestbucketlitellm # AWS Bucket Name for S3 + s3_region_name: us-west-2 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 75934ee1f1..0177c21907 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -21,6 +21,13 @@ else: Span = Any +class LiteLLMTeamRoles(enum.Enum): + # team admin + TEAM_ADMIN = "admin" + # team member + TEAM_MEMBER = "user" + + class LitellmUserRoles(str, enum.Enum): """ Admin Roles: @@ -335,6 +342,11 @@ class LiteLLMRoutes(enum.Enum): + sso_only_routes ) + self_managed_routes: List = [ + "/team/member_add", + "/team/member_delete", + ] # routes that manage their own allowed/disallowed logic + # class LiteLLMAllowedRoutes(LiteLLMBase): # """ @@ -1308,6 +1320,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): soft_budget: Optional[float] = None team_model_aliases: Optional[Dict] = None team_member_spend: Optional[float] = None + team_member: Optional[Member] = None team_metadata: Optional[Dict] = None # End User Params diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 58a2aa164f..fde5d5ca51 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -975,8 +975,6 @@ async def user_api_key_auth( if not _is_user_proxy_admin(user_obj=user_obj): # if non-admin if is_llm_api_route(route=route): pass - elif is_llm_api_route(route=request["route"].name): - pass elif ( route in LiteLLMRoutes.info_routes.value ): # check if user allowed to call an info route @@ -1046,11 +1044,16 @@ async def user_api_key_auth( status_code=status.HTTP_403_FORBIDDEN, detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}", ) + elif ( _user_role == LitellmUserRoles.INTERNAL_USER.value and route in LiteLLMRoutes.internal_user_routes.value ): pass + elif ( + route in LiteLLMRoutes.self_managed_routes.value + ): # routes that manage their own allowed/disallowed logic + pass else: user_role = "unknown" user_id = "unknown" diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 243ae18135..fa976690e6 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -285,14 +285,18 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str, return headers -def get_applied_guardrails_header(request_data: Dict) -> Optional[Dict]: +def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]: _metadata = request_data.get("metadata", None) or {} + headers = {} if "applied_guardrails" in _metadata: - return { - "x-litellm-applied-guardrails": ",".join(_metadata["applied_guardrails"]), - } + headers["x-litellm-applied-guardrails"] = ",".join( + _metadata["applied_guardrails"] + ) - return None + if "semantic-similarity" in _metadata: + headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"]) + + return headers def add_guardrail_to_applied_guardrails_header( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6c5d556c16..04706f0817 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -95,7 +95,9 @@ def convert_key_logging_metadata_to_callback( for var, value in data.callback_vars.items(): if team_callback_settings_obj.callback_vars is None: team_callback_settings_obj.callback_vars = {} - team_callback_settings_obj.callback_vars[var] = litellm.get_secret(value) + team_callback_settings_obj.callback_vars[var] = ( + litellm.utils.get_secret(value, default_value=value) or value + ) return team_callback_settings_obj @@ -130,7 +132,6 @@ def _get_dynamic_logging_metadata( data=AddTeamCallback(**item), team_callback_settings_obj=callback_settings_obj, ) - return callback_settings_obj diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index a0e020b11f..b570171182 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -119,6 +119,7 @@ async def new_user( http_request=Request( scope={"type": "http", "path": "/user/new"}, ), + user_api_key_dict=user_api_key_dict, ) if data.send_invite_email is True: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 1758b416dd..2e16b533c8 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -849,7 +849,7 @@ async def generate_key_helper_fn( } if ( - litellm.get_secret("DISABLE_KEY_NAME", False) == True + litellm.get_secret("DISABLE_KEY_NAME", False) is True ): # allow user to disable storing abbreviated key name (shown in UI, to help figure out which key spent how much) pass else: diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 815ab308c1..d3c2e3e839 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -30,7 +30,7 @@ from litellm.proxy._types import ( UpdateTeamRequest, UserAPIKeyAuth, ) -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.auth.user_api_key_auth import _is_user_proxy_admin, user_api_key_auth from litellm.proxy.management_helpers.utils import ( add_new_member, management_endpoint_wrapper, @@ -39,6 +39,16 @@ from litellm.proxy.management_helpers.utils import ( router = APIRouter() +def _is_user_team_admin( + user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable +) -> bool: + for member in team_obj.members_with_roles: + if member.user_id is not None and member.user_id == user_api_key_dict.user_id: + return True + + return False + + #### TEAM MANAGEMENT #### @router.post( "/team/new", @@ -417,6 +427,7 @@ async def team_member_add( If user doesn't exist, new user row will also be added to User Table + Only proxy_admin or admin of team, allowed to access this endpoint. ``` curl -X POST 'http://0.0.0.0:4000/team/member_add' \ @@ -465,6 +476,24 @@ async def team_member_add( complete_team_data = LiteLLM_TeamTable(**existing_team_row.model_dump()) + ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN + + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not _is_user_team_admin( + user_api_key_dict=user_api_key_dict, team_obj=complete_team_data + ) + ): + raise HTTPException( + status_code=403, + detail={ + "error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format( + "/team/member_add", + complete_team_data.team_id, + ) + }, + ) + if isinstance(data.member, Member): # add to team db new_member = data.member @@ -569,6 +598,23 @@ async def team_member_delete( ) existing_team_row = LiteLLM_TeamTable(**_existing_team_row.model_dump()) + ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN + + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not _is_user_team_admin( + user_api_key_dict=user_api_key_dict, team_obj=existing_team_row + ) + ): + raise HTTPException( + status_code=403, + detail={ + "error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format( + "/team/member_delete", existing_team_row.team_id + ) + }, + ) + ## DELETE MEMBER FROM TEAM new_team_members: List[Member] = [] for m in existing_team_row.members_with_roles: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 168a522dbd..5d8e221442 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,6 +1,7 @@ model_list: - model_name: multimodalembedding@001 litellm_params: + model: vertex_ai/multimodalembedding@001 vertex_project: "adroit-crow-413218" vertex_location: "us-central1" @@ -10,6 +11,3 @@ default_vertex_config: vertex_project: "adroit-crow-413218" vertex_location: "us-central1" vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cfc5fbf7fb..d07f63f53a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -149,7 +149,7 @@ from litellm.proxy.common_utils.admin_ui_utils import ( show_missing_vars_in_env, ) from litellm.proxy.common_utils.callback_utils import ( - get_applied_guardrails_header, + get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, initialize_callbacks_on_proxy, ) @@ -543,9 +543,9 @@ def get_custom_headers( ) headers.update(remaining_tokens_header) - applied_guardrails = get_applied_guardrails_header(request_data) - if applied_guardrails: - headers.update(applied_guardrails) + logging_caching_headers = get_logging_caching_headers(request_data) + if logging_caching_headers: + headers.update(logging_caching_headers) try: return { @@ -2784,26 +2784,29 @@ async def startup_event(): await custom_db_client.connect() if prisma_client is not None and master_key is not None: - # add master key to db if os.getenv("PROXY_ADMIN_ID", None) is not None: litellm_proxy_admin_name = os.getenv( "PROXY_ADMIN_ID", litellm_proxy_admin_name ) - asyncio.create_task( - generate_key_helper_fn( - request_type="user", - duration=None, - models=[], - aliases={}, - config={}, - spend=0, - token=master_key, - user_id=litellm_proxy_admin_name, - user_role=LitellmUserRoles.PROXY_ADMIN, - query_type="update_data", - update_key_values={"user_role": LitellmUserRoles.PROXY_ADMIN}, + if general_settings.get("disable_adding_master_key_hash_to_db") is True: + verbose_proxy_logger.info("Skipping writing master key hash to db") + else: + # add master key to db + asyncio.create_task( + generate_key_helper_fn( + request_type="user", + duration=None, + models=[], + aliases={}, + config={}, + spend=0, + token=master_key, + user_id=litellm_proxy_admin_name, + user_role=LitellmUserRoles.PROXY_ADMIN, + query_type="update_data", + update_key_values={"user_role": LitellmUserRoles.PROXY_ADMIN}, + ) ) - ) if prisma_client is not None and litellm.max_budget > 0: if litellm.budget_duration is None: diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index df4a2edfc3..a1a0b97339 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,4 +1,6 @@ import json +import os +import secrets import traceback from typing import Optional @@ -8,12 +10,30 @@ from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.proxy.utils import hash_token +def _is_master_key(api_key: str, _master_key: Optional[str]) -> bool: + if _master_key is None: + return False + + ## string comparison + is_master_key = secrets.compare_digest(api_key, _master_key) + if is_master_key: + return True + + ## hash comparison + is_master_key = secrets.compare_digest(api_key, hash_token(_master_key)) + if is_master_key: + return True + + return False + + def get_logging_payload( kwargs, response_obj, start_time, end_time, end_user_id: Optional[str] ) -> SpendLogsPayload: from pydantic import Json from litellm.proxy._types import LiteLLM_SpendLogs + from litellm.proxy.proxy_server import general_settings, master_key verbose_proxy_logger.debug( f"SpendTable: get_logging_payload - kwargs: {kwargs}\n\n" @@ -36,9 +56,15 @@ def get_logging_payload( usage = dict(usage) id = response_obj.get("id", kwargs.get("litellm_call_id")) api_key = metadata.get("user_api_key", "") - if api_key is not None and isinstance(api_key, str) and api_key.startswith("sk-"): - # hash the api_key - api_key = hash_token(api_key) + if api_key is not None and isinstance(api_key, str): + if api_key.startswith("sk-"): + # hash the api_key + api_key = hash_token(api_key) + if ( + _is_master_key(api_key=api_key, _master_key=master_key) + and general_settings.get("disable_adding_master_key_hash_to_db") is True + ): + api_key = "litellm_proxy_master_key" # use a known alias, if the user disabled storing master key in db _model_id = metadata.get("model_info", {}).get("id", "") _model_group = metadata.get("model_group", "") diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a2b09b4e69..a770177179 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -44,6 +44,7 @@ from litellm.proxy._types import ( DynamoDBArgs, LiteLLM_VerificationTokenView, LitellmUserRoles, + Member, ResetTeamBudgetRequest, SpendLogsMetadata, SpendLogsPayload, @@ -1395,6 +1396,7 @@ class PrismaClient: t.blocked AS team_blocked, t.team_alias AS team_alias, t.metadata AS team_metadata, + t.members_with_roles AS team_members_with_roles, tm.spend AS team_member_spend, m.aliases as team_model_aliases FROM "LiteLLM_VerificationToken" AS v @@ -1412,6 +1414,33 @@ class PrismaClient: response["team_models"] = [] if response["team_blocked"] is None: response["team_blocked"] = False + + team_member: Optional[Member] = None + if ( + response["team_members_with_roles"] is not None + and response["user_id"] is not None + ): + ## find the team member corresponding to user id + """ + [ + { + "role": "admin", + "user_id": "default_user_id", + "user_email": null + }, + { + "role": "user", + "user_id": null, + "user_email": "test@email.com" + } + ] + """ + for tm in response["team_members_with_roles"]: + if tm.get("user_id") is not None and response[ + "user_id" + ] == tm.get("user_id"): + team_member = Member(**tm) + response["team_member"] = team_member response = LiteLLM_VerificationTokenView( **response, last_refreshed_at=time.time() ) diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index e142f5696a..e30a9ed362 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -501,6 +501,8 @@ async def test_async_vertexai_streaming_response(): assert len(complete_response) > 0 except litellm.RateLimitError as e: pass + except litellm.APIConnectionError: + pass except litellm.Timeout as e: pass except litellm.InternalServerError as e: @@ -1558,6 +1560,16 @@ async def test_gemini_pro_json_schema_args_sent_httpx_openai_schema( "response_schema" in mock_call.call_args.kwargs["json"]["generationConfig"] ) + assert ( + "response_mime_type" + in mock_call.call_args.kwargs["json"]["generationConfig"] + ) + assert ( + mock_call.call_args.kwargs["json"]["generationConfig"][ + "response_mime_type" + ] + == "application/json" + ) else: assert ( "response_schema" diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 7df759add3..64196e5c56 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -1746,7 +1746,7 @@ async def test_qdrant_semantic_cache_acompletion(): litellm.cache = Cache( type="qdrant-semantic", _host_type="cloud", - qdrant_url=os.getenv("QDRANT_URL"), + qdrant_api_base=os.getenv("QDRANT_URL"), qdrant_api_key=os.getenv("QDRANT_API_KEY"), qdrant_collection_name="test_collection", similarity_threshold=0.8, @@ -1794,8 +1794,7 @@ async def test_qdrant_semantic_cache_acompletion_stream(): ] litellm.cache = Cache( type="qdrant-semantic", - qdrant_host_type="cloud", - qdrant_url=os.getenv("QDRANT_URL"), + qdrant_api_base=os.getenv("QDRANT_URL"), qdrant_api_key=os.getenv("QDRANT_API_KEY"), qdrant_collection_name="test_collection", similarity_threshold=0.8, diff --git a/litellm/tests/test_custom_callback_input.py b/litellm/tests/test_custom_callback_input.py index 01d0531833..a498b970bd 100644 --- a/litellm/tests/test_custom_callback_input.py +++ b/litellm/tests/test_custom_callback_input.py @@ -1252,3 +1252,45 @@ def test_standard_logging_payload(model, turn_off_message_logging): ] if turn_off_message_logging: assert "redacted-by-litellm" == slobject["messages"][0]["content"] + + +def test_standard_logging_payload_cache_hit(): + from litellm.types.utils import StandardLoggingPayload + + # sync completion + customHandler = CompletionCustomHandler() + litellm.callbacks = [customHandler] + + litellm.cache = Cache() + + _ = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + caching=True, + ) + + with patch.object( + customHandler, "log_success_event", new=MagicMock() + ) as mock_client: + _ = litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + caching=True, + ) + + time.sleep(2) + mock_client.assert_called_once() + + assert "standard_logging_object" in mock_client.call_args.kwargs["kwargs"] + assert ( + mock_client.call_args.kwargs["kwargs"]["standard_logging_object"] + is not None + ) + + standard_logging_object: StandardLoggingPayload = mock_client.call_args.kwargs[ + "kwargs" + ]["standard_logging_object"] + + assert standard_logging_object["cache_hit"] is True + assert standard_logging_object["response_cost"] == 0 + assert standard_logging_object["saved_cache_cost"] > 0 diff --git a/litellm/tests/test_proxy_server.py b/litellm/tests/test_proxy_server.py index 28f3aad632..94a548003d 100644 --- a/litellm/tests/test_proxy_server.py +++ b/litellm/tests/test_proxy_server.py @@ -909,7 +909,7 @@ async def test_create_team_member_add(prisma_client, new_member_method): await team_member_add( data=team_member_add_request, - user_api_key_dict=UserAPIKeyAuth(), + user_api_key_dict=UserAPIKeyAuth(user_role="proxy_admin"), http_request=Request( scope={"type": "http", "path": "/user/new"}, ), @@ -930,6 +930,172 @@ async def test_create_team_member_add(prisma_client, new_member_method): ) +@pytest.mark.parametrize("team_member_role", ["admin", "user"]) +@pytest.mark.parametrize("team_route", ["/team/member_add", "/team/member_delete"]) +@pytest.mark.asyncio +async def test_create_team_member_add_team_admin_user_api_key_auth( + prisma_client, team_member_role, team_route +): + import time + + from fastapi import Request + + from litellm.proxy._types import LiteLLM_TeamTableCachedObj, Member + from litellm.proxy.proxy_server import ( + ProxyException, + hash_token, + user_api_key_auth, + user_api_key_cache, + ) + + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + setattr(litellm, "max_internal_user_budget", 10) + setattr(litellm, "internal_user_budget_duration", "5m") + await litellm.proxy.proxy_server.prisma_client.connect() + user = f"ishaan {uuid.uuid4().hex}" + _team_id = "litellm-test-client-id-new" + user_key = "sk-12345678" + + valid_token = UserAPIKeyAuth( + team_id=_team_id, + token=hash_token(user_key), + team_member=Member(role=team_member_role, user_id=user), + last_refreshed_at=time.time(), + ) + user_api_key_cache.set_cache(key=hash_token(user_key), value=valid_token) + + team_obj = LiteLLM_TeamTableCachedObj( + team_id=_team_id, + blocked=False, + last_refreshed_at=time.time(), + metadata={"guardrails": {"modify_guardrails": False}}, + ) + + user_api_key_cache.set_cache(key="team_id:{}".format(_team_id), value=team_obj) + + setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + + ## TEST IF TEAM ADMIN ALLOWED TO CALL /MEMBER_ADD ENDPOINT + import json + + from starlette.datastructures import URL + + request = Request(scope={"type": "http"}) + request._url = URL(url=team_route) + + body = {} + json_bytes = json.dumps(body).encode("utf-8") + + request._body = json_bytes + + ## ALLOWED BY USER_API_KEY_AUTH + await user_api_key_auth(request=request, api_key="Bearer " + user_key) + + +@pytest.mark.parametrize("new_member_method", ["user_id", "user_email"]) +@pytest.mark.parametrize("user_role", ["admin", "user"]) +@pytest.mark.asyncio +async def test_create_team_member_add_team_admin( + prisma_client, new_member_method, user_role +): + """ + Relevant issue - https://github.com/BerriAI/litellm/issues/5300 + + Allow team admins to: + - Add and remove team members + - raise error if team member not an existing 'internal_user' + """ + import time + + from fastapi import Request + + from litellm.proxy._types import LiteLLM_TeamTableCachedObj, Member + from litellm.proxy.proxy_server import ( + HTTPException, + ProxyException, + hash_token, + user_api_key_auth, + user_api_key_cache, + ) + + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + setattr(litellm, "max_internal_user_budget", 10) + setattr(litellm, "internal_user_budget_duration", "5m") + await litellm.proxy.proxy_server.prisma_client.connect() + user = f"ishaan {uuid.uuid4().hex}" + _team_id = "litellm-test-client-id-new" + user_key = "sk-12345678" + + valid_token = UserAPIKeyAuth( + team_id=_team_id, + user_id=user, + token=hash_token(user_key), + last_refreshed_at=time.time(), + ) + user_api_key_cache.set_cache(key=hash_token(user_key), value=valid_token) + + team_obj = LiteLLM_TeamTableCachedObj( + team_id=_team_id, + blocked=False, + last_refreshed_at=time.time(), + members_with_roles=[Member(role=user_role, user_id=user)], + metadata={"guardrails": {"modify_guardrails": False}}, + ) + + user_api_key_cache.set_cache(key="team_id:{}".format(_team_id), value=team_obj) + + setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + if new_member_method == "user_id": + data = { + "team_id": _team_id, + "member": [{"role": "user", "user_id": user}], + } + elif new_member_method == "user_email": + data = { + "team_id": _team_id, + "member": [{"role": "user", "user_email": user}], + } + team_member_add_request = TeamMemberAddRequest(**data) + + with patch( + "litellm.proxy.proxy_server.prisma_client.db.litellm_usertable", + new_callable=AsyncMock, + ) as mock_litellm_usertable: + mock_client = AsyncMock() + mock_litellm_usertable.upsert = mock_client + mock_litellm_usertable.find_many = AsyncMock(return_value=None) + + try: + await team_member_add( + data=team_member_add_request, + user_api_key_dict=valid_token, + http_request=Request( + scope={"type": "http", "path": "/user/new"}, + ), + ) + except HTTPException as e: + if user_role == "user": + assert e.status_code == 403 + else: + raise e + + mock_client.assert_called() + + print(f"mock_client.call_args: {mock_client.call_args}") + print("mock_client.call_args.kwargs: {}".format(mock_client.call_args.kwargs)) + + assert ( + mock_client.call_args.kwargs["data"]["create"]["max_budget"] + == litellm.max_internal_user_budget + ) + assert ( + mock_client.call_args.kwargs["data"]["create"]["budget_duration"] + == litellm.internal_user_budget_duration + ) + + @pytest.mark.asyncio async def test_user_info_team_list(prisma_client): """Assert user_info for admin calls team_list function""" @@ -1116,8 +1282,8 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client): "callback_name": "langfuse", "callback_type": "success", "callback_vars": { - "langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY", - "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY", + "langfuse_public_key": "my-mock-public-key", + "langfuse_secret_key": "my-mock-secret-key", "langfuse_host": "https://us.cloud.langfuse.com", }, } @@ -1165,7 +1331,9 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client): assert "success_callback" in new_data assert new_data["success_callback"] == ["langfuse"] assert "langfuse_public_key" in new_data + assert new_data["langfuse_public_key"] == "my-mock-public-key" assert "langfuse_secret_key" in new_data + assert new_data["langfuse_secret_key"] == "my-mock-secret-key" @pytest.mark.asyncio diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a65701d2fd..8efbe5a114 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1218,6 +1218,7 @@ class StandardLoggingPayload(TypedDict): metadata: StandardLoggingMetadata cache_hit: Optional[bool] cache_key: Optional[str] + saved_cache_cost: Optional[float] request_tags: list end_user: Optional[str] requester_ip_address: Optional[str] diff --git a/litellm/utils.py b/litellm/utils.py index 7a3f977181..0c15cae53e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8622,7 +8622,9 @@ def get_secret( return secret_value_as_bool else: return secret - except: + except Exception: + if default_value is not None: + return default_value return secret except Exception as e: if default_value is not None: diff --git a/pyproject.toml b/pyproject.toml index ea2fcdb48a..ed49a29229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.44.1" +version = "1.44.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.44.1" +version = "1.44.2" version_files = [ "pyproject.toml:^version" ]