diff --git a/docs/my-website/docs/providers/vertex_batch.md b/docs/my-website/docs/providers/vertex_batch.md index 4eaa0d69d4..046c60f2eb 100644 --- a/docs/my-website/docs/providers/vertex_batch.md +++ b/docs/my-website/docs/providers/vertex_batch.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -## **Batch APIs** +# Vertex Batch APIs Just add the following Vertex env vars to your environment. diff --git a/docs/my-website/docs/providers/vertex_self_deployed.md b/docs/my-website/docs/providers/vertex_self_deployed.md index 98eff5d368..b7a71cdbd0 100644 --- a/docs/my-website/docs/providers/vertex_self_deployed.md +++ b/docs/my-website/docs/providers/vertex_self_deployed.md @@ -124,10 +124,7 @@ Deploy Gemma models on custom Vertex AI prediction endpoints with OpenAI-compati | Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) | | Required Parameter | `api_base` - Full prediction endpoint URL | -### Usage - - - +**Proxy Usage:** **1. Add to config.yaml** @@ -160,9 +157,7 @@ curl http://0.0.0.0:4000/v1/chat/completions \ }' ``` - - - +**SDK Usage:** ```python from litellm import completion @@ -176,5 +171,59 @@ response = completion( ) ``` - - +## MedGemma Models (Custom Endpoints) + +Deploy MedGemma models on custom Vertex AI prediction endpoints with OpenAI-compatible format. MedGemma models use the same `vertex_ai/gemma/` route. + +| Property | Details | +|----------|---------| +| Provider Route | `vertex_ai/gemma/{MODEL_NAME}` | +| Vertex Documentation | [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions) | +| Required Parameter | `api_base` - Full prediction endpoint URL | + +**Proxy Usage:** + +**1. Add to config.yaml** + +```yaml +model_list: + - model_name: medgemma-model + litellm_params: + model: vertex_ai/gemma/medgemma-2b-v1 + api_base: https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict + vertex_project: "my-project-id" + vertex_location: "us-central1" +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it** + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "medgemma-model", + "messages": [{"role": "user", "content": "What are the symptoms of hypertension?"}], + "max_tokens": 100 + }' +``` + +**SDK Usage:** + +```python +from litellm import completion + +response = completion( + model="vertex_ai/gemma/medgemma-2b-v1", + messages=[{"role": "user", "content": "What are the symptoms of hypertension?"}], + api_base="https://ENDPOINT.us-central1-PROJECT.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict", + vertex_project="my-project-id", + vertex_location="us-central1", +) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e8d002440d..00db20b8c4 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -423,6 +423,7 @@ const sidebars = { items: [ "providers/vertex", "providers/vertex_partner", + "providers/vertex_self_deployed", "providers/vertex_image", "providers/vertex_batch", ] diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 47cc429e79..24b53f0ba4 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -13,10 +13,9 @@ from typing import Any, Callable, Dict, List, Optional, Union, cast import httpx from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import LlmProviders, ModelResponse +from litellm.types.utils import ModelResponse class VertexGemmaConfig(OpenAIGPTConfig): @@ -88,6 +87,7 @@ class VertexGemmaConfig(OpenAIGPTConfig): # Remove params not needed/supported by Vertex Gemma openai_request.pop("model", None) openai_request.pop("stream", None) # Streaming not supported, will be faked client-side + openai_request.pop("stream_options", None) # Stream options not supported # Wrap in Vertex Gemma format return { diff --git a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index 3eed79fb16..9fbeb78712 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -309,3 +309,83 @@ class TestVertexGemmaCompletion: assert len(chunk.choices) > 0 assert chunk.choices[0].delta.content == "Streaming test response" + @pytest.mark.asyncio + async def test_acompletion_filters_stream_and_stream_options(self): + """ + Test that both stream and stream_options are filtered out from the request. + + Verifies that when stream=True and stream_options={'include_usage': True} are passed, + neither parameter is sent to the Vertex API since Vertex Gemma doesn't support them. + """ + # Mock Vertex response + mock_vertex_response = { + "deployedModelId": "1207280419999999999", + "model": "projects/993702345710/locations/us-central1/models/gemma-3-12b-it-1222199011122", + "modelDisplayName": "gemma-3-12b-it-1222199011122", + "modelVersionId": "1", + "predictions": { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": None, + "message": { + "content": "Test response", + "reasoning_content": None, + "role": "assistant", + "tool_calls": [], + }, + "stop_reason": None, + } + ], + "created": 1759863903, + "id": "chatcmpl-test", + "model": "google/gemma-3-12b-it", + "object": "chat.completion", + "prompt_logprobs": None, + "usage": { + "completion_tokens": 2, + "prompt_tokens": 10, + "prompt_tokens_details": None, + "total_tokens": 12, + }, + }, + } + + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client" + ) as mock_get_client: + mock_client = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = mock_vertex_response + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + # Call with both stream and stream_options + response = await litellm.acompletion( + model="vertex_ai/gemma/gemma-3-12b-it-1222199011122", + messages=[{"role": "user", "content": "Test"}], + stream=True, + stream_options={"include_usage": True}, + api_base="https://test.us-central1-project.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict", + vertex_project="PROJECT_ID", + vertex_location="us-central1", + ) + + # Verify the request sent to Vertex + call_args = mock_client.post.call_args + assert call_args is not None, "HTTP client was not called" + + request_data = call_args.kwargs["json"] + print("request body=", json.dumps(request_data, indent=4)) + instance = request_data["instances"][0] + + # Critical: Verify both stream and stream_options are NOT sent to Vertex API + assert "stream" not in instance, "stream parameter should not be sent to Vertex API" + assert "stream_options" not in instance, "stream_options parameter should not be sent to Vertex API" + + # Verify other parameters are present + assert "messages" in instance + assert instance["@requestFormat"] == "chatCompletions" +