From 6ce8f5692e7bad2ee7bb2e80730364d72f5af3cd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 5 Jul 2025 11:14:41 -0700 Subject: [PATCH] [Docs] vertex deepseek (#12345) * docs vertex * Vertex AI - Anthropic (Claude) --- docs/my-website/docs/providers/vertex.md | 643 +---------------- .../docs/providers/vertex_partner.md | 670 ++++++++++++++++++ .../release_notes/v1.74.0-stable/index.md | 24 +- docs/my-website/sidebars.js | 1 + 4 files changed, 682 insertions(+), 656 deletions(-) create mode 100644 docs/my-website/docs/providers/vertex_partner.md diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 6ec652f618..fda0cee862 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# VertexAI [Anthropic, Gemini, Model Garden] +# VertexAI [Gemini] ## Overview @@ -1208,534 +1208,6 @@ os.environ["VERTEXAI_LOCATION"] = "us-central1 # Your Location # set directly on module litellm.vertex_location = "us-central1 # Your Location ``` -## Anthropic -| Model Name | Function Call | -|------------------|--------------------------------------| -| claude-3-opus@20240229 | `completion('vertex_ai/claude-3-opus@20240229', messages)` | -| claude-3-5-sonnet@20240620 | `completion('vertex_ai/claude-3-5-sonnet@20240620', messages)` | -| claude-3-sonnet@20240229 | `completion('vertex_ai/claude-3-sonnet@20240229', messages)` | -| claude-3-haiku@20240307 | `completion('vertex_ai/claude-3-haiku@20240307', messages)` | -| claude-3-7-sonnet@20250219 | `completion('vertex_ai/claude-3-7-sonnet@20250219', messages)` | - -### Usage - - - - -```python -from litellm import completion -import os - -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" - -model = "claude-3-sonnet@20240229" - -vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] -vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] - -response = completion( - model="vertex_ai/" + model, - messages=[{"role": "user", "content": "hi"}], - temperature=0.7, - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, -) -print("\nModel Response", response) -``` - - - -**1. Add to config** - -```yaml -model_list: - - model_name: anthropic-vertex - litellm_params: - model: vertex_ai/claude-3-sonnet@20240229 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" - - model_name: anthropic-vertex - litellm_params: - model: vertex_ai/claude-3-sonnet@20240229 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "anthropic-vertex", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - - -### Usage - `thinking` / `reasoning_content` - - - - - -```python -from litellm import completion - -resp = completion( - model="vertex_ai/claude-3-7-sonnet-20250219", - messages=[{"role": "user", "content": "What is the capital of France?"}], - thinking={"type": "enabled", "budget_tokens": 1024}, -) - -``` - - - - - -1. Setup config.yaml - -```yaml -- model_name: claude-3-7-sonnet-20250219 - litellm_params: - model: vertex_ai/claude-3-7-sonnet-20250219 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -2. Start proxy - -```bash -litellm --config /path/to/config.yaml -``` - -3. Test it! - -```bash -curl http://0.0.0.0:4000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer " \ - -d '{ - "model": "claude-3-7-sonnet-20250219", - "messages": [{"role": "user", "content": "What is the capital of France?"}], - "thinking": {"type": "enabled", "budget_tokens": 1024} - }' -``` - - - - - -**Expected Response** - -```python -ModelResponse( - id='chatcmpl-c542d76d-f675-4e87-8e5f-05855f5d0f5e', - created=1740470510, - model='claude-3-7-sonnet-20250219', - object='chat.completion', - system_fingerprint=None, - choices=[ - Choices( - finish_reason='stop', - index=0, - message=Message( - content="The capital of France is Paris.", - role='assistant', - tool_calls=None, - function_call=None, - provider_specific_fields={ - 'citations': None, - 'thinking_blocks': [ - { - 'type': 'thinking', - 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.', - 'signature': 'EuYBCkQYAiJAy6...' - } - ] - } - ), - thinking_blocks=[ - { - 'type': 'thinking', - 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.', - 'signature': 'EuYBCkQYAiJAy6AGB...' - } - ], - reasoning_content='The capital of France is Paris. This is a very straightforward factual question.' - ) - ], - usage=Usage( - completion_tokens=68, - prompt_tokens=42, - total_tokens=110, - completion_tokens_details=None, - prompt_tokens_details=PromptTokensDetailsWrapper( - audio_tokens=None, - cached_tokens=0, - text_tokens=None, - image_tokens=None - ), - cache_creation_input_tokens=0, - cache_read_input_tokens=0 - ) -) -``` - - - -## Meta/Llama API - -| Model Name | Function Call | -|------------------|--------------------------------------| -| meta/llama-3.2-90b-vision-instruct-maas | `completion('vertex_ai/meta/llama-3.2-90b-vision-instruct-maas', messages)` | -| meta/llama3-8b-instruct-maas | `completion('vertex_ai/meta/llama3-8b-instruct-maas', messages)` | -| meta/llama3-70b-instruct-maas | `completion('vertex_ai/meta/llama3-70b-instruct-maas', messages)` | -| meta/llama3-405b-instruct-maas | `completion('vertex_ai/meta/llama3-405b-instruct-maas', messages)` | -| meta/llama-4-scout-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas', messages)` | -| meta/llama-4-scout-17-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-128b-16e-instruct-maas', messages)` | -| meta/llama-4-maverick-17b-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas',messages)` | -| meta/llama-4-maverick-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas',messages)` | - -### Usage - - - - -```python -from litellm import completion -import os - -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" - -model = "meta/llama3-405b-instruct-maas" - -vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] -vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] - -response = completion( - model="vertex_ai/" + model, - messages=[{"role": "user", "content": "hi"}], - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, -) -print("\nModel Response", response) -``` - - - -**1. Add to config** - -```yaml -model_list: - - model_name: anthropic-llama - litellm_params: - model: vertex_ai/meta/llama3-405b-instruct-maas - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" - - model_name: anthropic-llama - litellm_params: - model: vertex_ai/meta/llama3-405b-instruct-maas - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "anthropic-llama", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - -## Mistral API - -[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137) - -| Model Name | Function Call | -|------------------|--------------------------------------| -| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` | -| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` | -| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` | -| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` | -| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` | - -### Usage - - - - -```python -from litellm import completion -import os - -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" - -model = "mistral-large@2407" - -vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] -vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] - -response = completion( - model="vertex_ai/" + model, - messages=[{"role": "user", "content": "hi"}], - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, -) -print("\nModel Response", response) -``` - - - -**1. Add to config** - -```yaml -model_list: - - model_name: vertex-mistral - litellm_params: - model: vertex_ai/mistral-large@2407 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" - - model_name: vertex-mistral - litellm_params: - model: vertex_ai/mistral-large@2407 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "vertex-mistral", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - -### Usage - Codestral FIM - -Call Codestral on VertexAI via the OpenAI [`/v1/completion`](https://platform.openai.com/docs/api-reference/completions/create) endpoint for FIM tasks. - -Note: You can also call Codestral via `/chat/completion`. - - - - -```python -from litellm import completion -import os - -# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" -# OR run `!gcloud auth print-access-token` in your terminal - -model = "codestral@2405" - -vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] -vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] - -response = text_completion( - model="vertex_ai/" + model, - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, - prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():", - suffix="return True", # optional - temperature=0, # optional - top_p=1, # optional - max_tokens=10, # optional - min_tokens=10, # optional - seed=10, # optional - stop=["return"], # optional -) - -print("\nModel Response", response) -``` - - - -**1. Add to config** - -```yaml -model_list: - - model_name: vertex-codestral - litellm_params: - model: vertex_ai/codestral@2405 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" - - model_name: vertex-codestral - litellm_params: - model: vertex_ai/codestral@2405 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl -X POST 'http://0.0.0.0:4000/completions' \ - -H 'Authorization: Bearer sk-1234' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "vertex-codestral", # 👈 the 'model_name' in config - "prompt": "def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():", - "suffix":"return True", # optional - "temperature":0, # optional - "top_p":1, # optional - "max_tokens":10, # optional - "min_tokens":10, # optional - "seed":10, # optional - "stop":["return"], # optional - }' -``` - - - - - -## AI21 Models - -| Model Name | Function Call | -|------------------|--------------------------------------| -| jamba-1.5-mini@001 | `completion(model='vertex_ai/jamba-1.5-mini@001', messages)` | -| jamba-1.5-large@001 | `completion(model='vertex_ai/jamba-1.5-large@001', messages)` | - -### Usage - - - - -```python -from litellm import completion -import os - -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" - -model = "meta/jamba-1.5-mini@001" - -vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] -vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] - -response = completion( - model="vertex_ai/" + model, - messages=[{"role": "user", "content": "hi"}], - vertex_ai_project=vertex_ai_project, - vertex_ai_location=vertex_ai_location, -) -print("\nModel Response", response) -``` - - - -**1. Add to config** - -```yaml -model_list: - - model_name: jamba-1.5-mini - litellm_params: - model: vertex_ai/jamba-1.5-mini@001 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" - - model_name: jamba-1.5-large - litellm_params: - model: vertex_ai/jamba-1.5-large@001 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-west-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "jamba-1.5-large", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - ## Gemini Pro | Model Name | Function Call | @@ -1832,119 +1304,6 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \ - - -## Model Garden - -:::tip - -All OpenAI compatible models from Vertex Model Garden are supported. - -::: - -#### Using Model Garden - -**Almost all Vertex Model Garden models are OpenAI compatible.** - - - - - -| Property | Details | -|----------|---------| -| Provider Route | `vertex_ai/openai/{MODEL_ID}` | -| Vertex Documentation | [SDK for Deploy & OpenAI Chat Completions](https://github.com/GoogleCloudPlatform/generative-ai/blob/main/open-models/get_started_with_model_garden_sdk.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) | -| Supported Operations | `/chat/completions`, `/embeddings` | - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/openai/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: llama3-1-8b-instruct - litellm_params: - model: vertex_ai/openai/5464397967697903616 - vertex_ai_project: "my-test-project" - vertex_ai_location: "us-east-1" -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - - - - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - - - ## Gemini Pro Vision | Model Name | Function Call | |------------------|--------------------------------------| diff --git a/docs/my-website/docs/providers/vertex_partner.md b/docs/my-website/docs/providers/vertex_partner.md new file mode 100644 index 0000000000..09e08942f9 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_partner.md @@ -0,0 +1,670 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + +# Vertex AI - Anthropic, DeepSeek, Model Garden + +## Supported Partner Providers + +| Provider | LiteLLM Route | Vertex Documentation | +|----------|---------------|---------------| +| Anthropic (Claude) | `vertex_ai/claude-*` | [Vertex AI - Anthropic Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude) | +| DeepSeek | `vertex_ai/deepseek-ai/{MODEL}` | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) | +| Meta/Llama | `vertex_ai/meta/{MODEL}` | [Vertex AI - Meta Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/llama) | +| Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) | +| AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) | +| Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) | + +## Vertex AI - Anthropic (Claude) + +| Model Name | Function Call | +|------------------|--------------------------------------| +| claude-3-opus@20240229 | `completion('vertex_ai/claude-3-opus@20240229', messages)` | +| claude-3-5-sonnet@20240620 | `completion('vertex_ai/claude-3-5-sonnet@20240620', messages)` | +| claude-3-sonnet@20240229 | `completion('vertex_ai/claude-3-sonnet@20240229', messages)` | +| claude-3-haiku@20240307 | `completion('vertex_ai/claude-3-haiku@20240307', messages)` | +| claude-3-7-sonnet@20250219 | `completion('vertex_ai/claude-3-7-sonnet@20250219', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "claude-3-sonnet@20240229" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + temperature=0.7, + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: anthropic-vertex + litellm_params: + model: vertex_ai/claude-3-sonnet@20240229 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: anthropic-vertex + litellm_params: + model: vertex_ai/claude-3-sonnet@20240229 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "anthropic-vertex", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + + +#### Usage - `thinking` / `reasoning_content` + + + + + +```python +from litellm import completion + +resp = completion( + model="vertex_ai/claude-3-7-sonnet-20250219", + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 1024}, +) + +``` + + + + + +1. Setup config.yaml + +```yaml +- model_name: claude-3-7-sonnet-20250219 + litellm_params: + model: vertex_ai/claude-3-7-sonnet-20250219 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "claude-3-7-sonnet-20250219", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "thinking": {"type": "enabled", "budget_tokens": 1024} + }' +``` + + + + + +**Expected Response** + +```python +ModelResponse( + id='chatcmpl-c542d76d-f675-4e87-8e5f-05855f5d0f5e', + created=1740470510, + model='claude-3-7-sonnet-20250219', + object='chat.completion', + system_fingerprint=None, + choices=[ + Choices( + finish_reason='stop', + index=0, + message=Message( + content="The capital of France is Paris.", + role='assistant', + tool_calls=None, + function_call=None, + provider_specific_fields={ + 'citations': None, + 'thinking_blocks': [ + { + 'type': 'thinking', + 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.', + 'signature': 'EuYBCkQYAiJAy6...' + } + ] + } + ), + thinking_blocks=[ + { + 'type': 'thinking', + 'thinking': 'The capital of France is Paris. This is a very straightforward factual question.', + 'signature': 'EuYBCkQYAiJAy6AGB...' + } + ], + reasoning_content='The capital of France is Paris. This is a very straightforward factual question.' + ) + ], + usage=Usage( + completion_tokens=68, + prompt_tokens=42, + total_tokens=110, + completion_tokens_details=None, + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=None, + cached_tokens=0, + text_tokens=None, + image_tokens=None + ), + cache_creation_input_tokens=0, + cache_read_input_tokens=0 + ) +) +``` + +## VertexAI DeepSeek + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/deepseek-ai/{MODEL}` | +| Vertex Documentation | [Vertex AI - DeepSeek Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/deepseek) | + +#### Usage + +**LiteLLM Supports all Vertex AI DeepSeek Models.** Ensure you use the `vertex_ai/deepseek-ai/` prefix for all Vertex AI DeepSeek models. + +| Model Name | Usage | +|------------------|------------------------------| +| vertex_ai/deepseek-ai/deepseek-r1-0528-maas | `completion('vertex_ai/deepseek-ai/deepseek-r1-0528-maas', messages)` | + + +## VertexAI Meta/Llama API + +| Model Name | Function Call | +|------------------|--------------------------------------| +| meta/llama-3.2-90b-vision-instruct-maas | `completion('vertex_ai/meta/llama-3.2-90b-vision-instruct-maas', messages)` | +| meta/llama3-8b-instruct-maas | `completion('vertex_ai/meta/llama3-8b-instruct-maas', messages)` | +| meta/llama3-70b-instruct-maas | `completion('vertex_ai/meta/llama3-70b-instruct-maas', messages)` | +| meta/llama3-405b-instruct-maas | `completion('vertex_ai/meta/llama3-405b-instruct-maas', messages)` | +| meta/llama-4-scout-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas', messages)` | +| meta/llama-4-scout-17-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-scout-128b-16e-instruct-maas', messages)` | +| meta/llama-4-maverick-17b-128e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas',messages)` | +| meta/llama-4-maverick-17b-16e-instruct-maas | `completion('vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas',messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "meta/llama3-405b-instruct-maas" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: anthropic-llama + litellm_params: + model: vertex_ai/meta/llama3-405b-instruct-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: anthropic-llama + litellm_params: + model: vertex_ai/meta/llama3-405b-instruct-maas + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "anthropic-llama", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +## VertexAI Mistral API + +[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137) + +| Model Name | Function Call | +|------------------|--------------------------------------| +| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` | +| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` | +| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` | +| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` | +| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "mistral-large@2407" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: vertex-mistral + litellm_params: + model: vertex_ai/mistral-large@2407 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: vertex-mistral + litellm_params: + model: vertex_ai/mistral-large@2407 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "vertex-mistral", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + +#### Usage - Codestral FIM + +Call Codestral on VertexAI via the OpenAI [`/v1/completion`](https://platform.openai.com/docs/api-reference/completions/create) endpoint for FIM tasks. + +Note: You can also call Codestral via `/chat/completion`. + + + + +```python +from litellm import completion +import os + +# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" +# OR run `!gcloud auth print-access-token` in your terminal + +model = "codestral@2405" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = text_completion( + model="vertex_ai/" + model, + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, + prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():", + suffix="return True", # optional + temperature=0, # optional + top_p=1, # optional + max_tokens=10, # optional + min_tokens=10, # optional + seed=10, # optional + stop=["return"], # optional +) + +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: vertex-codestral + litellm_params: + model: vertex_ai/codestral@2405 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: vertex-codestral + litellm_params: + model: vertex_ai/codestral@2405 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl -X POST 'http://0.0.0.0:4000/completions' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "vertex-codestral", # 👈 the 'model_name' in config + "prompt": "def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():", + "suffix":"return True", # optional + "temperature":0, # optional + "top_p":1, # optional + "max_tokens":10, # optional + "min_tokens":10, # optional + "seed":10, # optional + "stop":["return"], # optional + }' +``` + + + + + +## VertexAI AI21 Models + +| Model Name | Function Call | +|------------------|--------------------------------------| +| jamba-1.5-mini@001 | `completion(model='vertex_ai/jamba-1.5-mini@001', messages)` | +| jamba-1.5-large@001 | `completion(model='vertex_ai/jamba-1.5-large@001', messages)` | + +#### Usage + + + + +```python +from litellm import completion +import os + +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "" + +model = "meta/jamba-1.5-mini@001" + +vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"] +vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"] + +response = completion( + model="vertex_ai/" + model, + messages=[{"role": "user", "content": "hi"}], + vertex_ai_project=vertex_ai_project, + vertex_ai_location=vertex_ai_location, +) +print("\nModel Response", response) +``` + + + +**1. Add to config** + +```yaml +model_list: + - model_name: jamba-1.5-mini + litellm_params: + model: vertex_ai/jamba-1.5-mini@001 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" + - model_name: jamba-1.5-large + litellm_params: + model: vertex_ai/jamba-1.5-large@001 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-west-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "jamba-1.5-large", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + +## Model Garden + +:::tip + +All OpenAI compatible models from Vertex Model Garden are supported. + +::: + +#### Using Model Garden + +**Almost all Vertex Model Garden models are OpenAI compatible.** + + + + + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/openai/{MODEL_ID}` | +| Vertex Documentation | [SDK for Deploy & OpenAI Chat Completions](https://github.com/GoogleCloudPlatform/generative-ai/blob/main/open-models/get_started_with_model_garden_sdk.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) | +| Supported Operations | `/chat/completions`, `/embeddings` | + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/openai/", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: llama3-1-8b-instruct + litellm_params: + model: vertex_ai/openai/5464397967697903616 + vertex_ai_project: "my-test-project" + vertex_ai_location: "us-east-1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + + + + + + + + +```python +from litellm import completion +import os + +## set ENV variables +os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + diff --git a/docs/my-website/release_notes/v1.74.0-stable/index.md b/docs/my-website/release_notes/v1.74.0-stable/index.md index 4d50e04816..cfc4fc70ea 100644 --- a/docs/my-website/release_notes/v1.74.0-stable/index.md +++ b/docs/my-website/release_notes/v1.74.0-stable/index.md @@ -58,16 +58,21 @@ pip install litellm==1.74.0.post1 This release brings significant performance improvements to the Python SDK with 2.3 seconds faster import times. We've refactored the initialization process to reduce startup overhead, making LiteLLM more efficient for applications that need quick initialization. This is a major improvement for applications that need to initialize LiteLLM quickly. - - - --- ## New Models / Updated Models -### New Providers -### Updated Models +#### Features +- **[🆕 GitHub Copilot](../../docs/providers/github_copilot)** - Use GitHub Copilot API with LiteLLM - [PR](https://github.com/BerriAI/litellm/pull/12325), [Get Started](../../docs/providers/github_copilot) +- **[🆕 VertexAI DeepSeek](../../docs/providers/vertex)** - Add support for VertexAI DeepSeek models - [PR](https://github.com/BerriAI/litellm/pull/12312), [Get Started](../../docs/providers/vertex) +- **[Azure AI](../../docs/providers/azure_ai)** + - Add azure_ai cohere rerank v3.5 - [PR](https://github.com/BerriAI/litellm/pull/12283) +- **[Vertex AI](../../docs/providers/vertex)** + - Add size parameter support for image generation - [PR](https://github.com/BerriAI/litellm/pull/12292) +- **[Custom LLM](../../docs/providers/custom_llm_server)** + - Pass through extra_ properties on "custom" llm provider - [PR](https://github.com/BerriAI/litellm/pull/12185) + #### Bugs - **[Mistral](../../docs/providers/mistral)** - Fix transform_response handling for empty string content - [PR](https://github.com/BerriAI/litellm/pull/12202) @@ -87,15 +92,6 @@ This release brings significant performance improvements to the Python SDK with - **[Hugging Face](../../docs/providers/huggingface)** - Fix Hugging Face tests - [PR](https://github.com/BerriAI/litellm/pull/12286) -#### Features -- **[🆕 GitHub Copilot](../../docs/providers/github_copilot)** - Use GitHub Copilot API with LiteLLM -- **[🆕 VertexAI DeepSeek](../../docs/providers/vertex)** - Add support for VertexAI DeepSeek models - [PR](https://github.com/BerriAI/litellm/pull/12312) -- **[Azure AI](../../docs/providers/azure_ai)** - - Add azure_ai cohere rerank v3.5 - [PR](https://github.com/BerriAI/litellm/pull/12283) -- **[Vertex AI](../../docs/providers/vertex)** - - Add size parameter support for image generation - [PR](https://github.com/BerriAI/litellm/pull/12292) -- **[Custom LLM](../../docs/providers/custom_llm_server)** - - Pass through extra_ properties on "custom" llm provider - [PR](https://github.com/BerriAI/litellm/pull/12185) --- diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8dc7aa35eb..7c2aa9ff26 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -360,6 +360,7 @@ const sidebars = { label: "Vertex AI", items: [ "providers/vertex", + "providers/vertex_partner", "providers/vertex_image", ] },