From 76107bfb6b87be77f4fd426c2e0724bc87a11c27 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 24 May 2025 14:11:15 -0700 Subject: [PATCH] docs(litellm_managed_files.md): add tutorial on managing file permissions --- .../docs/proxy/litellm_managed_files.md | 161 ++++++++++++++++-- 1 file changed, 148 insertions(+), 13 deletions(-) diff --git a/docs/my-website/docs/proxy/litellm_managed_files.md b/docs/my-website/docs/proxy/litellm_managed_files.md index de4bd1b9bf..63e8e51740 100644 --- a/docs/my-website/docs/proxy/litellm_managed_files.md +++ b/docs/my-website/docs/proxy/litellm_managed_files.md @@ -4,7 +4,8 @@ import Image from '@theme/IdealImage'; # [BETA] LiteLLM Managed Files -Reuse the same file across different providers. +- Reuse the same file across different providers. +- Prevent users from seeing files they don't have access to on `list` and `retrieve` calls. :::info @@ -15,22 +16,18 @@ Available via the `litellm[proxy]` package or any `litellm` docker image. ::: -| Feature | Description | Comments | +| Property | Value | Comments | | --- | --- | --- | | Proxy | ✅ | | -| SDK | ❌ | Requires postgres DB for storing file ids | +| SDK | ❌ | Requires postgres DB for storing file ids. | | Available across all providers | ✅ | | +| Supported endpoints | `/chat/completions`, `/batch`, `/fine_tuning` | | - - -Limitations of LiteLLM Managed Files: -- Only works for `/chat/completions` and `/batch` requests. - -Follow [here](https://github.com/BerriAI/litellm/discussions/9632) for multiple models, batches support. +## Usage ### 1. Setup config.yaml -``` +```yaml model_list: - model_name: "gemini-2.0-flash" litellm_params: @@ -41,6 +38,10 @@ model_list: litellm_params: model: gpt-4o-mini api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 # alternatively use the env var - LITELLM_MASTER_KEY + database_url: "postgresql://:@:/" # alternatively use the env var - DATABASE_URL ``` ### 2. Start proxy @@ -219,8 +220,120 @@ print(completion.choices[0].message) ``` +## File Permissions -### Supported Endpoints +Prevent users from seeing files they don't have access to on `list` and `retrieve` calls. + +### 1. Setup config.yaml + +```yaml +model_list: + - model_name: "gpt-4o-mini-openai" + litellm_params: + model: gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 # alternatively use the env var - LITELLM_MASTER_KEY + database_url: "postgresql://:@:/" # alternatively use the env var - DATABASE_URL +``` + +### 2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +### 3. Issue a key to the user + +Let's create a user with the id `user_123`. + +```bash +curl -L -X POST 'http://0.0.0.0:4000/user/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"models": ["gpt-4o-mini-openai"], "user_id": "user_123"}' +``` + +Get the key from the response. + +```json +{ + "key": "sk-..." +} +``` + +### 4. User creates a file + +#### 4a. Create a file + +```jsonl +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +``` + +#### 4b. Upload the file + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://0.0.0.0:4000", + api_key="sk-...", # 👈 Use the key you generated in step 3 + max_retries=0 +) + +# Upload file +finetuning_input_file = client.files.create( + file=open("./fine_tuning.jsonl", "rb"), # {"model": "azure-gpt-4o"} <-> {"model": "gpt-4o-my-special-deployment"} + purpose="fine-tune", + extra_body={"target_model_names": "gpt-4.1-openai"} # 👈 Tells litellm which regions/projects to write the file in. +) +print(finetuning_input_file) # file.id = "litellm_proxy/..." = {"model_name": {"deployment_id": "deployment_file_id"}} +``` + +### 5. User retrieves a file + + + + +```python +from openai import OpenAI + +... # User created file (3b) + +file = client.files.retrieve( + file_id=finetuning_input_file.id +) + +print(file) # File retrieved successfully +``` + + + + +```python +```python +from openai import OpenAI + +... # User created file (3b) + +try: + file = client.files.retrieve( + file_id="bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCwyYTgzOWIyYS03YzI1LTRiNTUtYTUxYS1lZjdhODljNzZkMzU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1iYXRjaA" + ) +except Exception as e: + print(e) # User does not have access to this file + +``` + + + + + + + +## Supported Endpoints #### Create a file - `/files` @@ -264,7 +377,23 @@ client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-1234", max_retries=0 file = client.files.delete(file_id=file.id) ``` -### FAQ +#### List files - `/files` + +```python +client = OpenAI(base_url="http://0.0.0.0:4000", api_key="sk-1234", max_retries=0) + +files = client.files.list(extra_body={"target_model_names": "gpt-4o-mini-openai"}) + +print(files) # All files user has created +``` + +Pre-GA Limitations on List Files: + - No multi-model support: Just 1 model name is supported for now. + - No multi-deployment support: Just 1 deployment of the model is supported for now (e.g. if you have 2 deployments with the `gpt-4o-mini-openai` public model name, it will pick one and return all files on that deployment). + +Pre-GA Limitations will be fixed before GA of the Managed Files feature. + +## FAQ **1. Does LiteLLM store the file?** @@ -278,7 +407,13 @@ LiteLLM stores a mapping of the litellm file id to the model-specific file id in When a file is deleted, LiteLLM deletes the mapping from the postgres DB, and the files on each provider. -### Architecture +**4. Can a user call a file id that was created by another user?** + +No, as of `v1.71.2` users can only view/edit/delete files they have created. + + + +## Architecture