From c23b2c502333236eb1ca6e8bb1aa6e66fc972726 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 12 Nov 2025 18:23:31 -0800 Subject: [PATCH 01/19] Config Guardrails should not be deletable from table (#16540) --- .../src/components/guardrails.tsx | 6 +- .../guardrails/guardrail_table.test.tsx | 56 ++++++++++++++++++ .../components/guardrails/guardrail_table.tsx | 58 +++++++++---------- .../guardrails/pii_components.test.tsx | 40 +++++++++++++ .../components/guardrails/pii_components.tsx | 2 + .../guardrails/pii_configuration.test.tsx | 20 +++++++ .../guardrails/pii_configuration.tsx | 15 ++--- .../src/components/guardrails/types.ts | 6 ++ 8 files changed, 161 insertions(+), 42 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/guardrails/guardrail_table.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/pii_components.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/pii_configuration.test.tsx diff --git a/ui/litellm-dashboard/src/components/guardrails.tsx b/ui/litellm-dashboard/src/components/guardrails.tsx index 23aec34c40..3861545f8c 100644 --- a/ui/litellm-dashboard/src/components/guardrails.tsx +++ b/ui/litellm-dashboard/src/components/guardrails.tsx @@ -8,6 +8,7 @@ import { isAdminRole } from "@/utils/roles"; import GuardrailInfoView from "./guardrails/guardrail_info"; import GuardrailTestPlayground from "./guardrails/GuardrailTestPlayground"; import NotificationsManager from "./molecules/notifications_manager"; +import { Guardrail, GuardrailDefinitionLocation } from "./guardrails/types"; interface GuardrailsPanelProps { accessToken: string | null; @@ -25,14 +26,15 @@ interface GuardrailItem { guardrail_info: Record | null; created_at?: string; updated_at?: string; + guardrail_definition_location: GuardrailDefinitionLocation; } interface GuardrailsResponse { - guardrails: GuardrailItem[]; + guardrails: Guardrail[]; } const GuardrailsPanel: React.FC = ({ accessToken, userRole }) => { - const [guardrailsList, setGuardrailsList] = useState([]); + const [guardrailsList, setGuardrailsList] = useState([]); const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isLoading, setIsLoading] = useState(false); const [isDeleting, setIsDeleting] = useState(false); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.test.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.test.tsx new file mode 100644 index 0000000000..dcc6767735 --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.test.tsx @@ -0,0 +1,56 @@ +import GuardrailTable from "./guardrail_table"; +import { render } from "@testing-library/react"; +import { describe, it, expect } from "vitest"; +import { GuardrailDefinitionLocation } from "./types"; +describe("GuardrailTable", () => { + it("should render", () => { + const { getByText } = render( + {}} + accessToken={null} + onGuardrailUpdated={() => {}} + onGuardrailClick={() => {}} + />, + ); + expect(getByText("Guardrail ID")).toBeInTheDocument(); + expect(getByText("Name")).toBeInTheDocument(); + expect(getByText("Provider")).toBeInTheDocument(); + expect(getByText("Mode")).toBeInTheDocument(); + expect(getByText("Default On")).toBeInTheDocument(); + expect(getByText("Created At")).toBeInTheDocument(); + expect(getByText("Updated At")).toBeInTheDocument(); + }); + + it("should not allow deletion of config guardrails", () => { + const { getByTestId } = render( + {}} + accessToken={null} + onGuardrailUpdated={() => {}} + onGuardrailClick={() => {}} + />, + ); + + const deleteGuardrailButton = getByTestId("config-delete-icon"); + expect(deleteGuardrailButton).toBeInTheDocument(); + expect(deleteGuardrailButton).toHaveClass("cursor-not-allowed text-gray-400"); + expect(deleteGuardrailButton).toHaveAttribute( + "title", + "Config guardrail cannot be deleted on the dashboard. Please delete it from the config file.", + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index 4857cdd31e..6a2476960f 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -13,24 +13,10 @@ import { } from "@tanstack/react-table"; import { getGuardrailLogoAndName, guardrail_provider_map } from "./guardrail_info_helpers"; import EditGuardrailForm from "./edit_guardrail_form"; - -interface GuardrailItem { - guardrail_id?: string; - guardrail_name: string | null; - litellm_params: { - guardrail: string; - mode: string; - default_on: boolean; - pii_entities_config?: { [key: string]: string }; - [key: string]: any; - }; - guardrail_info: Record | null; - created_at?: string; - updated_at?: string; -} +import { Guardrail, GuardrailDefinitionLocation } from "./types"; interface GuardrailTableProps { - guardrailsList: GuardrailItem[]; + guardrailsList: Guardrail[]; isLoading: boolean; onDeleteClick: (guardrailId: string, guardrailName: string) => void; accessToken: string | null; @@ -50,7 +36,7 @@ const GuardrailTable: React.FC = ({ }) => { const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); const [editModalVisible, setEditModalVisible] = useState(false); - const [selectedGuardrail, setSelectedGuardrail] = useState(null); + const [selectedGuardrail, setSelectedGuardrail] = useState(null); // Format date helper function const formatDate = (dateString?: string) => { @@ -59,7 +45,7 @@ const GuardrailTable: React.FC = ({ return date.toLocaleString(); }; - const handleEditClick = (guardrail: GuardrailItem) => { + const handleEditClick = (guardrail: Guardrail) => { setSelectedGuardrail(guardrail); setEditModalVisible(true); }; @@ -70,7 +56,7 @@ const GuardrailTable: React.FC = ({ onGuardrailUpdated(); }; - const columns: ColumnDef[] = [ + const columns: ColumnDef[] = [ { header: "Guardrail ID", accessorKey: "guardrail_id", @@ -176,18 +162,32 @@ const GuardrailTable: React.FC = ({ header: "", cell: ({ row }) => { const guardrail = row.original; + const isConfigGuardrail = guardrail.guardrail_definition_location === GuardrailDefinitionLocation.CONFIG; return (
- - guardrail.guardrail_id && - onDeleteClick(guardrail.guardrail_id, guardrail.guardrail_name || "Unnamed Guardrail") - } - className="cursor-pointer hover:text-red-500" - tooltip="Delete guardrail" - /> + {isConfigGuardrail ? ( + + + + ) : ( + + guardrail.guardrail_id && + onDeleteClick(guardrail.guardrail_id, guardrail.guardrail_name || "Unnamed Guardrail") + } + className="cursor-pointer hover:text-red-500" + tooltip="Delete guardrail" + /> + )}
); }, diff --git a/ui/litellm-dashboard/src/components/guardrails/pii_components.test.tsx b/ui/litellm-dashboard/src/components/guardrails/pii_components.test.tsx new file mode 100644 index 0000000000..3accbf13bd --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/pii_components.test.tsx @@ -0,0 +1,40 @@ +import { render } from "@testing-library/react"; +import { describe, it, expect } from "vitest"; +import { CategoryFilter, QuickActions, PiiEntityList } from "./pii_components"; +import type { PiiEntityCategory } from "./types"; + +describe("CategoryFilter", () => { + it("should render", () => { + const emptyCategories: PiiEntityCategory[] = []; + const { getByText } = render( + {}} />, + ); + expect(getByText("Filter by category")).toBeInTheDocument(); + }); +}); + +describe("QuickActions", () => { + it("should render", () => { + const { getByText } = render( + {}} onUnselectAll={() => {}} hasSelectedEntities={false} />, + ); + expect(getByText("Quick Actions")).toBeInTheDocument(); + }); +}); + +describe("PiiEntityList", () => { + it("should render", () => { + const { getByText } = render( + {}} + onActionSelect={() => {}} + entityToCategoryMap={new Map()} + />, + ); + expect(getByText("No PII types match your filter criteria")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/guardrails/pii_components.tsx b/ui/litellm-dashboard/src/components/guardrails/pii_components.tsx index 1b744cad31..e3b3926b99 100644 --- a/ui/litellm-dashboard/src/components/guardrails/pii_components.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/pii_components.tsx @@ -83,6 +83,7 @@ export const QuickActions: React.FC = ({ onSelectAll, onUnsel + Add Callback From 8bf491c9391ca517052c3371bb032c95de14ba77 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 12 Nov 2025 18:44:21 -0800 Subject: [PATCH 07/19] [Fix] /spend/logs/ui Access Control (#16446) * RBAC for /spend/logs/ui * Addressing comments --- .../spend_management_endpoints.py | 68 ++++ .../test_spend_management_endpoints.py | 291 ++++++++++++++++++ 2 files changed, 359 insertions(+) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 51ece5dd84..b60acd1428 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -18,6 +18,10 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( ) from litellm.proxy.utils import handle_exception_on_proxy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting +from litellm.proxy.management_endpoints.common_utils import ( + _is_user_team_admin, + _user_has_admin_view, +) if TYPE_CHECKING: from litellm.proxy.proxy_server import PrismaClient @@ -1749,6 +1753,28 @@ async def ui_view_spend_logs( # noqa: PLR0915 where_conditions["spend"]["gte"] = min_spend if max_spend is not None: where_conditions["spend"]["lte"] = max_spend + is_admin_view = _is_admin_view_safe(user_api_key_dict=user_api_key_dict) + if not is_admin_view: + if team_id is not None: + can_view_team = await _can_team_member_view_log( + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + team_id=team_id, + ) + if not can_view_team: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": "Not authorized to view team spend for team_id={}".format( + team_id + ) + }, + ) + where_conditions["team_id"] = team_id + else: + if _can_user_view_spend_log(user_api_key_dict=user_api_key_dict): + where_conditions["user"] = user_api_key_dict.user_id + where_conditions.pop("team_id", None) # Calculate skip value for pagination skip = (page - 1) * page_size @@ -2990,3 +3016,45 @@ def _build_status_filter_condition(status_filter: Optional[str]) -> Dict[str, An return {"OR": [{"status": {"equals": "success"}}, {"status": None}]} else: return {"status": {"equals": status_filter}} + + +def _is_admin_view_safe(user_api_key_dict: UserAPIKeyAuth) -> bool: + """ + Safely determine if the current user has admin view permissions. + Wraps the underlying check and defaults to False on any exception. + """ + try: + return _user_has_admin_view(user_api_key_dict=user_api_key_dict) + except Exception: + return False + + +async def _can_team_member_view_log( + prisma_client, + user_api_key_dict: UserAPIKeyAuth, + team_id: Optional[str], +) -> bool: + """ + Check if the requesting user can view spend logs for the given team. + Returns True only if the team exists and the user is a team admin. + """ + if team_id is None: + return False + team_obj = await prisma_client.db.litellm_teamtable.find_unique( + where={"team_id": team_id} + ) + if team_obj is None: + return False + return _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj) + + +def _can_user_view_spend_log(user_api_key_dict: UserAPIKeyAuth) -> bool: + """ + Check if the requesting user can view their own spend logs. + """ + user_role = user_api_key_dict.user_role + user_id = user_api_key_dict.user_id + return user_role in ( + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ) and user_id is not None diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index c9c602e1cc..3a9229b3c4 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -21,6 +21,169 @@ from litellm.proxy.proxy_server import app, prisma_client from litellm.proxy.spend_tracking import spend_management_endpoints from litellm.router import Router from litellm.types.utils import BudgetConfig +from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles, Member +from litellm.proxy.spend_tracking import spend_management_endpoints +import litellm.proxy.proxy_server as ps + +@pytest.mark.asyncio +async def test_is_admin_view_safe_true(monkeypatch): + # Force underlying check to return True + monkeypatch.setattr( + spend_management_endpoints, "_user_has_admin_view", lambda user_api_key_dict: True + ) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user") + assert spend_management_endpoints._is_admin_view_safe(auth) is True + + +@pytest.mark.asyncio +async def test_is_admin_view_safe_false(monkeypatch): + # Force underlying check to return False + monkeypatch.setattr( + spend_management_endpoints, "_user_has_admin_view", lambda user_api_key_dict: False + ) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + assert spend_management_endpoints._is_admin_view_safe(auth) is False + + +@pytest.mark.asyncio +async def test_is_admin_view_safe_exception(monkeypatch): + # Ensure exceptions are swallowed and return False + def raise_err(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(spend_management_endpoints, "_user_has_admin_view", raise_err) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + assert spend_management_endpoints._is_admin_view_safe(auth) is False + + +@pytest.mark.asyncio +async def test_can_team_member_view_log_none_team_id(): + # team_id=None should immediately return False + class MockPrisma: + class DB: + class TeamTable: + async def find_unique(self, where: dict): + return None + + def __init__(self): + self.litellm_teamtable = self.TeamTable() + + def __init__(self): + self.db = self.DB() + + prisma = MockPrisma() + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + allowed = await spend_management_endpoints._can_team_member_view_log( + prisma, auth, None + ) + assert allowed is False + + +@pytest.mark.asyncio +async def test_can_team_member_view_log_team_not_found(monkeypatch): + # Non-existent team should return False + class MockPrisma: + class DB: + class TeamTable: + async def find_unique(self, where: dict): + return None + + def __init__(self): + self.litellm_teamtable = self.TeamTable() + + def __init__(self): + self.db = self.DB() + + prisma = MockPrisma() + # Even if admin check would return True, no team means False + monkeypatch.setattr( + spend_management_endpoints, "_is_user_team_admin", lambda user_api_key_dict, team_obj: True + ) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + allowed = await spend_management_endpoints._can_team_member_view_log( + prisma, auth, "team_x" + ) + assert allowed is False + + +@pytest.mark.asyncio +async def test_can_team_member_view_log_not_admin(monkeypatch): + # Existing team but caller is not a team admin -> False + class MockTeam: + pass + + class MockPrisma: + class DB: + class TeamTable: + async def find_unique(self, where: dict): + return MockTeam() + + def __init__(self): + self.litellm_teamtable = self.TeamTable() + + def __init__(self): + self.db = self.DB() + + prisma = MockPrisma() + monkeypatch.setattr( + spend_management_endpoints, "_is_user_team_admin", lambda user_api_key_dict, team_obj: False + ) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + allowed = await spend_management_endpoints._can_team_member_view_log( + prisma, auth, "team_x" + ) + assert allowed is False + + +@pytest.mark.asyncio +async def test_can_team_member_view_log_admin(monkeypatch): + # Existing team and caller is team admin -> True + class MockTeam: + pass + + class MockPrisma: + class DB: + class TeamTable: + async def find_unique(self, where: dict): + return MockTeam() + + def __init__(self): + self.litellm_teamtable = self.TeamTable() + + def __init__(self): + self.db = self.DB() + + prisma = MockPrisma() + monkeypatch.setattr( + spend_management_endpoints, "_is_user_team_admin", lambda user_api_key_dict, team_obj: True + ) + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="user_1") + allowed = await spend_management_endpoints._can_team_member_view_log( + prisma, auth, "team_x" + ) + assert allowed is True + + +def test_can_user_view_spend_log_true_for_internal_user(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="u1") + assert spend_management_endpoints._can_user_view_spend_log(auth) is True + + +def test_can_user_view_spend_log_true_for_internal_view_only(): + auth = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, user_id="u1" + ) + assert spend_management_endpoints._can_user_view_spend_log(auth) is True + + +def test_can_user_view_spend_log_false_without_user_id(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id=None) + assert spend_management_endpoints._can_user_view_spend_log(auth) is False + + +def test_can_user_view_spend_log_false_for_other_roles(): + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + assert spend_management_endpoints._can_user_view_spend_log(auth) is False ignored_keys = [ "request_id", @@ -255,6 +418,134 @@ async def test_ui_view_spend_logs_with_team_id(client, monkeypatch): assert data["data"][0]["team_id"] == "team1" +@pytest.mark.asyncio +async def test_ui_view_spend_logs_internal_user_scoped_without_user_id(client, monkeypatch): + """ + Internal users should only be able to view their own spend even if user_id is not provided. + """ + # Mock spend logs for 2 users + mock_spend_logs = [ + {"id": "log1", "request_id": "req1", "api_key": "sk-test-key", "user": "internal_user_1", "team_id": "team1", "spend": 0.05, "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-3.5-turbo"}, + {"id": "log2", "request_id": "req2", "api_key": "sk-test-key", "user": "internal_user_2", "team_id": "team1", "spend": 0.10, "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-4"}, + ] + + # Prisma client mock that filters by "user" where condition + class MockDB: + async def find_many(self, *args, **kwargs): + where = kwargs.get("where", {}) + if "user" in where and where["user"] == "internal_user_1": + return [mock_spend_logs[0]] + return mock_spend_logs + + async def count(self, *args, **kwargs): + where = kwargs.get("where", {}) + if "user" in where and where["user"] == "internal_user_1": + return 1 + return len(mock_spend_logs) + + class MockPrismaClient: + def __init__(self): + self.db = MockDB() + self.db.litellm_spendlogs = self.db + + mock_prisma_client = MockPrismaClient() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + # Override auth dependency to return INTERNAL_USER with specific user_id + # Override using the function reference attached to the running app module + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user_1" + ) + + try: + start_date = (datetime.datetime.now(timezone.utc) - datetime.timedelta(days=7)).strftime("%Y-%m-%d %H:%M:%S") + end_date = datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + # No user_id provided; should auto-scope to authenticated internal user's own id + response = client.get( + "/spend/logs/ui", + params={"start_date": start_date, "end_date": end_date}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert len(data["data"]) == 1 + assert data["data"][0]["user"] == "internal_user_1" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_ui_view_spend_logs_team_admin_can_view_team_spend(client, monkeypatch): + """ + Team admins should be able to view team-wide spend when team_id is provided. + """ + # Mock spend logs for two teams + mock_spend_logs = [ + {"id": "log1", "request_id": "req1", "api_key": "sk-test-key", "user": "member1", "team_id": "team_admin_team", "spend": 0.05, "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-3.5-turbo"}, + {"id": "log2", "request_id": "req2", "api_key": "sk-test-key", "user": "member2", "team_id": "team_other", "spend": 0.10, "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-4"}, + ] + + class MockDB: + async def find_many(self, *args, **kwargs): + where = kwargs.get("where", {}) + if "team_id" in where and where["team_id"] == "team_admin_team": + return [mock_spend_logs[0]] + return mock_spend_logs + + async def count(self, *args, **kwargs): + where = kwargs.get("where", {}) + if "team_id" in where and where["team_id"] == "team_admin_team": + return 1 + return len(mock_spend_logs) + + class MockPrismaClient: + def __init__(self): + self.db = MockDB() + self.db.litellm_spendlogs = self.db + # Team lookup for RBAC check + class TeamTable: + def __init__(self): + # user "admin_user" is team admin + self.members_with_roles = [Member(user_id="admin_user", role="admin")] + + async def find_unique(where: dict): + if where == {"team_id": "team_admin_team"}: + return TeamTable() + return None + + self.db.litellm_teamtable = self + self.litellm_teamtable = self + self.find_unique = find_unique + + mock_prisma_client = MockPrismaClient() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + + # Override auth dependency to return INTERNAL_USER (who is a team admin via team.members_with_roles) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, user_id="admin_user" + ) + + try: + start_date = (datetime.datetime.now(timezone.utc) - datetime.timedelta(days=7)).strftime("%Y-%m-%d %H:%M:%S") + end_date = datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + response = client.get( + "/spend/logs/ui", + params={"team_id": "team_admin_team", "start_date": start_date, "end_date": end_date}, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 1 + assert len(data["data"]) == 1 + assert data["data"][0]["team_id"] == "team_admin_team" + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + @pytest.mark.asyncio async def test_ui_view_spend_logs_pagination(client, monkeypatch): # Create a larger set of mock data for pagination testing From 018bd2e039fb33f6a6015bfde30c30f7c2e9d116 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 13 Nov 2025 08:18:27 +0530 Subject: [PATCH 08/19] Add Gemini image edit support (#16430) * Add gemini image edit support * fix lint errors * fix lint errors * fix lint errors * Add docs --- docs/my-website/docs/image_edits.md | 84 +++++++- docs/my-website/docs/providers/gemini.md | 2 +- litellm/cost_calculator.py | 1 + .../litellm_core_utils/llm_cost_calc/utils.py | 13 ++ litellm/llms/gemini/image_edit/__init__.py | 11 + .../llms/gemini/image_edit/cost_calculator.py | 35 ++++ .../llms/gemini/image_edit/transformation.py | 197 ++++++++++++++++++ litellm/utils.py | 4 + model_prices_and_context_window.json | 1 + .../test_gemini_image_edit_transformation.py | 149 +++++++++++++ 10 files changed, 493 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/gemini/image_edit/__init__.py create mode 100644 litellm/llms/gemini/image_edit/cost_calculator.py create mode 100644 litellm/llms/gemini/image_edit/transformation.py create mode 100644 tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md index 84dddd5e4a..9a53da510f 100644 --- a/docs/my-website/docs/image_edits.md +++ b/docs/my-website/docs/image_edits.md @@ -14,9 +14,9 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Supported operations | Create image edits | Single and multiple images supported | -| Supported LiteLLM SDK Versions | 1.63.8+ | | -| Supported LiteLLM Proxy Versions | 1.71.1+ | | -| Supported LLM providers | **OpenAI** | Currently only `openai` is supported | +| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ | +| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ | +| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family | #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/) @@ -149,6 +149,54 @@ for i, image_data in enumerate(response.data): print(f"Image {i+1}: {image_data.url}") ``` +``` + + + + + +#### Basic Image Edit +```python showLineNumbers title="Gemini Image Edit" +import base64 +import os +from litellm import image_edit + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = image_edit( + model="gemini/gemini-2.5-flash-image", + image=open("original_image.png", "rb"), + prompt="Add aurora borealis to the night sky", + size="1792x1024", # mapped to aspectRatio=16:9 for Gemini +) + +edited_image_bytes = base64.b64decode(response.data[0].b64_json) +with open("edited_image.png", "wb") as f: + f.write(edited_image_bytes) +``` + +#### Multiple Images Edit +```python showLineNumbers title="Gemini Multiple Images Edit" +import base64 +import os +from litellm import image_edit + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = image_edit( + model="gemini/gemini-2.5-flash-image", + image=[ + open("scene.png", "rb"), + open("style_reference.png", "rb"), + ], + prompt="Blend the reference style into the scene while keeping the subject sharp.", +) + +for idx, image_obj in enumerate(response.data): + with open(f"gemini_edit_{idx}.png", "wb") as f: + f.write(base64.b64decode(image_obj.b64_json)) +``` + @@ -224,6 +272,36 @@ curl -X POST "http://localhost:4000/v1/images/edits" \ -F "response_format=url" ``` +``` + + + + + +1. Add the Gemini image edit model to your `config.yaml`: +```yaml showLineNumbers title="Gemini Proxy Configuration" +model_list: + - model_name: gemini-image-edit + litellm_params: + model: gemini/gemini-2.5-flash-image + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start the LiteLLM proxy server: +```bash showLineNumbers title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml +``` + +3. Make an image edit request (Gemini responses are base64-only): +```bash showLineNumbers title="Gemini Proxy Image Edit" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=gemini-image-edit" \ + -F "image=@original_image.png" \ + -F "prompt=Add a warm golden-hour glow to the scene" \ + -F "size=1024x1024" +``` + diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 31d3a491f4..1d483cd489 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem'; | Provider Route on LiteLLM | `gemini/` | | Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) | | API Endpoint for Provider | https://generativelanguage.googleapis.com | -| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md) | +| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) | | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d4a4c441eb..d1c7ede655 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -943,6 +943,7 @@ def completion_cost( # noqa: PLR0915 n=n, size=size, optional_params=optional_params, + call_type=call_type, ) elif ( call_type == CallTypes.create_video.value diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index b55065352d..eff5376e49 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -640,6 +640,7 @@ class CostCalculatorUtils: n: Optional[int] = None, size: Optional[str] = None, optional_params: Optional[dict] = None, + call_type: Optional[str] = None, ) -> float: """ Route the image generation cost calculator based on the custom_llm_provider @@ -713,6 +714,18 @@ class CostCalculatorUtils: image_response=completion_response, ) elif custom_llm_provider == litellm.LlmProviders.GEMINI.value: + if call_type in ( + CallTypes.image_edit.value, + CallTypes.aimage_edit.value, + ): + from litellm.llms.gemini.image_edit.cost_calculator import ( + cost_calculator as gemini_image_edit_cost_calculator, + ) + + return gemini_image_edit_cost_calculator( + model=model, + image_response=completion_response, + ) from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_cost_calculator, ) diff --git a/litellm/llms/gemini/image_edit/__init__.py b/litellm/llms/gemini/image_edit/__init__.py new file mode 100644 index 0000000000..6181015b81 --- /dev/null +++ b/litellm/llms/gemini/image_edit/__init__.py @@ -0,0 +1,11 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .transformation import GeminiImageEditConfig +from .cost_calculator import cost_calculator + +__all__ = ["GeminiImageEditConfig", "get_gemini_image_edit_config", "cost_calculator"] + + +def get_gemini_image_edit_config(model: str) -> BaseImageEditConfig: + return GeminiImageEditConfig() + diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py new file mode 100644 index 0000000000..31f35345d8 --- /dev/null +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -0,0 +1,35 @@ +""" +Gemini Image Edit Cost Calculator +""" + +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + Gemini image edit cost calculator. + + Mirrors image generation pricing: charge per returned image based on + model metadata (`output_cost_per_image`). + """ + model_info = litellm.get_model_info( + model=model, + custom_llm_provider="gemini", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + + if not isinstance(image_response, ImageResponse): + raise ValueError( + f"image_response must be of type ImageResponse got type={type(image_response)}" + ) + + num_images = len(image_response.data or []) + return output_cost_per_image * num_images + diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py new file mode 100644 index 0000000000..830c58a006 --- /dev/null +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -0,0 +1,197 @@ +import base64 +from io import BufferedReader, BytesIO +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class GeminiImageEditConfig(BaseImageEditConfig): + DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" + SUPPORTED_PARAMS: List[str] = ["size"] + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + return mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + final_api_key: Optional[str] = api_key or get_secret_str("GEMINI_API_KEY") + if not final_api_key: + raise ValueError("GEMINI_API_KEY is not set") + + headers["x-goog-api-key"] = final_api_key + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + base_url = api_base or get_secret_str("GEMINI_API_BASE") or self.DEFAULT_BASE_URL + base_url = base_url.rstrip("/") + return f"{base_url}/models/{model}:generateContent" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + inline_parts = self._prepare_inline_image_parts(image) + if not inline_parts: + raise ValueError("Gemini image edit requires at least one image.") + + contents = [ + { + "parts": inline_parts + [{"text": prompt}], + } + ] + + request_body: Dict[str, Any] = {"contents": contents} + + generation_config: Dict[str, Any] = {} + + if "aspectRatio" in image_edit_optional_request_params: + generation_config["aspectRatio"] = image_edit_optional_request_params[ + "aspectRatio" + ] + + if generation_config: + request_body["generationConfig"] = generation_config + + empty_files = cast(RequestFiles, []) + return request_body, empty_files + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + candidates = response_json.get("candidates", []) + data_list: List[ImageObject] = [] + + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + inline_data = part.get("inlineData") + if inline_data and inline_data.get("data"): + data_list.append( + ImageObject( + b64_json=inline_data["data"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_inline_image_parts( + self, image: Union[FileTypes, List[FileTypes]] + ) -> List[Dict[str, Any]]: + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + inline_parts: List[Dict[str, Any]] = [] + for img in images: + if img is None: + continue + + mime_type = ImageEditRequestUtils.get_image_content_type(img) + image_bytes = self._read_all_bytes(img) + inline_parts.append( + { + "inlineData": { + "mimeType": mime_type, + "data": base64.b64encode(image_bytes).decode("utf-8"), + } + } + ) + + return inline_parts + + def _read_all_bytes(self, image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, BufferedReader): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + raise ValueError("Unsupported image type for Gemini image edit.") \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 4b9c1d9051..7f87a800b0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7720,6 +7720,10 @@ class ProviderConfigManager: from litellm.llms.azure_ai.image_edit import get_azure_ai_image_edit_config return get_azure_ai_image_edit_config(model) + elif LlmProviders.GEMINI == provider: + from litellm.llms.gemini.image_edit import get_gemini_image_edit_config + + return get_gemini_image_edit_config(model) elif LlmProviders.LITELLM_PROXY == provider: from litellm.llms.litellm_proxy.image_edit.transformation import ( LiteLLMProxyImageEditConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aa7774c43e..e81c00dd62 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -11670,6 +11670,7 @@ "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, + "supports_reasoning": false, "max_images_per_prompt": 3000, "max_input_tokens": 32768, "max_output_tokens": 32768, diff --git a/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py new file mode 100644 index 0000000000..2732bf1595 --- /dev/null +++ b/tests/test_litellm/llms/gemini/image_edit/test_gemini_image_edit_transformation.py @@ -0,0 +1,149 @@ +import base64 +import json +from io import BytesIO +from typing import Dict +from unittest.mock import MagicMock + +import httpx +import pytest + +from litellm.llms.gemini.image_edit.transformation import GeminiImageEditConfig + + +class TestGeminiImageEditTransformation: + def setup_method(self) -> None: + self.config = GeminiImageEditConfig() + self.model = "gemini-2.5-flash-image-preview" + self.prompt = "Enhance this photo with a dramatic night sky." + self.logging_obj = MagicMock() + + def test_map_openai_params(self) -> None: + optional_params: Dict[str, object] = { + "size": "1792x1024", + "response_format": "b64_json", + "quality": "high", + } + + mapped = self.config.map_openai_params( + image_edit_optional_params=optional_params, # type: ignore[arg-type] + model=self.model, + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert "response_format" not in mapped + assert "quality" not in mapped + + def test_transform_image_edit_request(self) -> None: + image_bytes = b"fake_image_data" + image = BytesIO(image_bytes) + optional_params = { + "sampleCount": 2, + "aspectRatio": "16:9", + } + + request_body, files = self.config.transform_image_edit_request( + model=self.model, + prompt=self.prompt, + image=[image], # Gemini pipeline passes list of images + image_edit_optional_request_params=optional_params, + litellm_params=MagicMock(), + headers={}, + ) + + assert files == [] + + parts = request_body["contents"][0]["parts"] + assert parts[-1]["text"] == self.prompt + + inline_data = parts[0]["inlineData"] + assert inline_data["mimeType"] == "image/png" + assert base64.b64decode(inline_data["data"]) == image_bytes + + generation_config = request_body["generationConfig"] + assert generation_config["aspectRatio"] == "16:9" + + def test_transform_image_edit_request_multiple_images(self) -> None: + image_one = BytesIO(b"image_one") + image_two = BytesIO(b"image_two") + + request_body, files = self.config.transform_image_edit_request( + model=self.model, + prompt=self.prompt, + image=[image_one, image_two], + image_edit_optional_request_params={}, + litellm_params=MagicMock(), + headers={}, + ) + + assert files == [] + parts = request_body["contents"][0]["parts"] + + assert len(parts) == 3 # two images + text prompt + assert parts[-1]["text"] == self.prompt + assert base64.b64decode(parts[0]["inlineData"]["data"]) == b"image_one" + assert base64.b64decode(parts[1]["inlineData"]["data"]) == b"image_two" + + def test_transform_image_edit_response(self) -> None: + response_payload = { + "candidates": [ + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": base64.b64encode(b"image-one").decode("utf-8"), + } + } + ] + } + }, + { + "content": { + "parts": [ + { + "inlineData": { + "mimeType": "image/png", + "data": base64.b64encode(b"image-two").decode("utf-8"), + } + } + ] + } + }, + ] + } + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = response_payload + mock_response.status_code = 200 + mock_response.headers = {} + + image_response = self.config.transform_image_edit_response( + model=self.model, + raw_response=mock_response, + logging_obj=self.logging_obj, + ) + + assert image_response.data is not None + assert len(image_response.data) == 2 + assert image_response.data[0].b64_json == base64.b64encode(b"image-one").decode( + "utf-8" + ) + assert image_response.data[1].b64_json == base64.b64encode(b"image-two").decode( + "utf-8" + ) + + def test_transform_image_edit_request_without_image_raises(self) -> None: + optional_params = {} + + with pytest.raises(ValueError): + self.config.transform_image_edit_request( + model=self.model, + prompt=self.prompt, + image=[], + image_edit_optional_request_params=optional_params, + litellm_params=MagicMock(), + headers={}, + ) + From 049d45ea9017626ebe460dec23a7a5302d53597a Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:01:38 -0300 Subject: [PATCH 09/19] fix(gemini): Preserve non-ASCII characters in function call arguments (#16550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #16533 Before this fix, non-ASCII characters (Japanese, Spanish, Chinese, etc.) in function call arguments were being escaped as Unicode sequences. Example: - Before: "やあ" → "\u3084\u3042" - After: "やあ" → "やあ" (preserved) Changes: - Add ensure_ascii=False to json.dumps() in _transform_parts() - Add test for Japanese and Spanish Unicode character preservation This is not a breaking change as both formats are equivalent in JSON. The fix improves readability and aligns with OpenAI's behavior. --- .../vertex_and_google_ai_studio_gemini.py | 2 +- tests/llm_translation/test_gemini.py | 69 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index b8370d5fef..67b7fa749a 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1022,7 +1022,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "functionCall" in part: _function_chunk = ChatCompletionToolCallFunctionChunk( name=part["functionCall"]["name"], - arguments=json.dumps(part["functionCall"]["args"]), + arguments=json.dumps(part["functionCall"]["args"], ensure_ascii=False), ) if is_function_call is True: function = _function_chunk diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index b7628a4903..3b33b6a346 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1135,3 +1135,72 @@ def test_gemini_embedding(): ) print("response: ", response) assert response is not None + + +def test_gemini_function_args_preserve_unicode(): + """ + Test for Issue #16533: Gemini function call arguments should preserve non-ASCII characters + https://github.com/BerriAI/litellm/issues/16533 + + Before fix: "や" becomes "\u3084" + After fix: "や" stays as "や" + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig + + # Test Japanese characters + parts = [ + { + "functionCall": { + "name": "send_message", + "args": { + "message": "やあ", # Japanese "hello" + "recipient": "たけし" # Japanese name + } + } + } + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts, + cumulative_tool_call_idx=0, + is_function_call=False + ) + + arguments_str = tools[0]['function']['arguments'] + parsed_args = json.loads(arguments_str) + + # Verify characters are preserved + assert parsed_args["message"] == "やあ", "Japanese characters should be preserved" + assert parsed_args["recipient"] == "たけし", "Japanese characters should be preserved" + + # Verify no Unicode escape sequences in raw string + assert "\\u" not in arguments_str, "Should not contain Unicode escape sequences" + assert "やあ" in arguments_str, "Original Japanese characters should be in the string" + assert "たけし" in arguments_str, "Original Japanese characters should be in the string" + + # Test Spanish characters + parts_spanish = [ + { + "functionCall": { + "name": "send_message", + "args": { + "message": "¡Hola! ¿Cómo estás?", + "recipient": "José" + } + } + } + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_spanish, + cumulative_tool_call_idx=0, + is_function_call=False + ) + + arguments_str = tools[0]['function']['arguments'] + parsed_args = json.loads(arguments_str) + + assert parsed_args["message"] == "¡Hola! ¿Cómo estás?" + assert parsed_args["recipient"] == "José" + assert "\\u" not in arguments_str + assert "José" in arguments_str From c017f665e04d97626ac05377ef9e8f054775089e Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:40:11 -0300 Subject: [PATCH 10/19] docs(openai): Document reasoning_effort summary field options (#16549) Related to PR #16210 which fixed automatic summary field addition Changes: - Document reasoning_effort string vs dict formats - Add summary field options (auto, detailed, concise) - Add table of supported reasoning_effort values by GPT-5 model - Clarify model-specific support and limitations - Note that summary field requires org verification The previous implementation automatically added summary field causing 400 errors for unverified orgs. Now users can opt-in by passing reasoning_effort as dict with explicit summary field. --- docs/my-website/docs/providers/openai.md | 71 ++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index f9831c6d8b..51ebc881d2 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -410,6 +410,77 @@ Expected Response: ``` +### Advanced: Using `reasoning_effort` with `summary` field + +By default, `reasoning_effort` accepts a string value (`"low"`, `"medium"`, `"high"`, `"minimal"`) and only sets the effort level without including a reasoning summary. + +To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI. + + + +```python +# Option 1: String format (default - no summary) +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="high" # Only sets effort level +) + +# Option 2: Dict format (with optional summary - requires org verification) +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort={"effort": "high", "summary": "auto"} # "auto", "detailed", or "concise" (not all supported by all models) +) +``` + + + +```bash +# Option 1: String format (default - no summary) +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "openai/responses/gpt-5-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": "high" +}' + +# Option 2: Dict format (with optional summary - requires org verification) +# summary options: "auto", "detailed", or "concise" (not all supported by all models) +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "openai/responses/gpt-5-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": {"effort": "high", "summary": "auto"} +}' +``` + + + +**Summary field options:** +- `"auto"`: System automatically determines the appropriate summary level based on the model +- `"concise"`: Provides a shorter summary (not supported by GPT-5 series models) +- `"detailed"`: Offers a comprehensive reasoning summary + +**Note:** GPT-5 series models support `"auto"` and `"detailed"`, but do not support `"concise"`. O-series models (o3-pro, o4-mini, o3) support all three options. Some models like o3-mini and o1 do not support reasoning summaries at all. + +**Supported `reasoning_effort` values by model:** + +| Model | Default (when not set) | Supported Values | +|-------|----------------------|------------------| +| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` | +| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` | +| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | +| `gpt-5-pro` | `high` | `high` only | + +**Note:** `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column. + +See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements. + ## OpenAI Chat Completion to Responses API Bridge Call any Responses API model from OpenAI's `/chat/completions` endpoint. From 491f57a3490ca041c7eb7095b0f23ff1d478f315 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:41:07 -0300 Subject: [PATCH 11/19] feat: Add support for reasoning_effort="none" for Gemini models (#16548) Implements support for reasoning_effort="none" parameter for Gemini models, providing significant cost savings (up to 96% cheaper) by disabling thinking budget while maintaining response quality. Changes: - Added "supports_reasoning": true to gemini-2.0-flash-thinking-exp-01-21 in model config - Implemented mapping for reasoning_effort="none" to thinkingConfig {thinkingBudget: 0, includeThoughts: false} - Added unit test to verify the mapping works correctly Performance impact: - Without reasoning_effort: ~313 tokens - With reasoning_effort="none": ~12 tokens (96% cheaper) Closes #16420 Co-authored-by: Krish Dholakia --- docs/my-website/docs/providers/gemini.md | 27 ++++++++++++++----- .../vertex_and_google_ai_studio_gemini.py | 5 ++++ model_prices_and_context_window.json | 2 ++ tests/llm_translation/test_gemini.py | 19 +++++++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 1d483cd489..c5014fc2ff 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -64,16 +64,21 @@ response = completion( LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362) -Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests. +**Cost Optimization:** Use `reasoning_effort="none"` (OpenAI standard) for significant cost savings - up to 96% cheaper. [Google's docs](https://ai.google.dev/gemini-api/docs/openai) + +:::info +Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. +::: **Mapping** -| reasoning_effort | thinking | -| ---------------- | -------- | -| "disable" | "budget_tokens": 0 | -| "low" | "budget_tokens": 1024 | -| "medium" | "budget_tokens": 2048 | -| "high" | "budget_tokens": 4096 | +| reasoning_effort | thinking | Notes | +| ---------------- | -------- | ----- | +| "none" | "budget_tokens": 0, "includeThoughts": false | 💰 **Recommended for cost optimization** - OpenAI-compatible, always 0 | +| "disable" | "budget_tokens": DEFAULT (0), "includeThoughts": false | LiteLLM-specific, configurable via env var | +| "low" | "budget_tokens": 1024 | | +| "medium" | "budget_tokens": 2048 | | +| "high" | "budget_tokens": 4096 | | @@ -81,6 +86,14 @@ Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini ```python from litellm import completion +# Cost-optimized: Use reasoning_effort="none" for best pricing +resp = completion( + model="gemini/gemini-2.0-flash-thinking-exp-01-21", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="none", # Up to 96% cheaper! +) + +# Or use other levels: "low", "medium", "high" resp = completion( model="gemini/gemini-2.5-flash-preview-04-17", messages=[{"role": "user", "content": "What is the capital of France?"}], diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 67b7fa749a..cbd8cf320c 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -567,6 +567,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "thinkingBudget": DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET, "includeThoughts": False, } + elif reasoning_effort == "none": + return { + "thinkingBudget": 0, + "includeThoughts": False, + } else: raise ValueError(f"Invalid reasoning effort: {reasoning_effort}") diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e81c00dd62..4eac8335e6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9963,6 +9963,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, @@ -11568,6 +11569,7 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, diff --git a/tests/llm_translation/test_gemini.py b/tests/llm_translation/test_gemini.py index 3b33b6a346..1065509dd4 100644 --- a/tests/llm_translation/test_gemini.py +++ b/tests/llm_translation/test_gemini.py @@ -1137,6 +1137,25 @@ def test_gemini_embedding(): assert response is not None +def test_reasoning_effort_none_mapping(): + """ + Test that reasoning_effort='none' correctly maps to thinkingConfig. + Related issue: https://github.com/BerriAI/litellm/issues/16420 + """ + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + # Test reasoning_effort="none" mapping + result = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget( + reasoning_effort="none", + model="gemini-2.0-flash-thinking-exp-01-21", + ) + + assert result is not None + assert result["thinkingBudget"] == 0 + assert result["includeThoughts"] is False + def test_gemini_function_args_preserve_unicode(): """ Test for Issue #16533: Gemini function call arguments should preserve non-ASCII characters From e1c607e22afc081b8804ff4f54ee33537d2a8232 Mon Sep 17 00:00:00 2001 From: Lucas Sugi Date: Thu, 13 Nov 2025 00:46:34 -0300 Subject: [PATCH 12/19] feat: Add headers to VLLM Passthrough requests [Log success Events] (#16532) --- litellm/proxy/pass_through_endpoints/pass_through_endpoints.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 2f101b13d8..3eee47f201 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -521,6 +521,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): "url": str(request.url), "method": request.method, "body": copy.copy(_parsed_body), # use copy instead of deepcopy + "headers": request.headers, }, }, "call_type": "pass_through_endpoint", From 555d7b8be89847f0089a93677a2463a90774e592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Thu, 13 Nov 2025 16:17:36 +0000 Subject: [PATCH 13/19] feat(bedrock): Add bearer token authentication support for AgentCore (#16556) --- .../bedrock/chat/agentcore/transformation.py | 180 +++++++------ .../llm_translation/test_bedrock_agentcore.py | 241 ++++++++++++++++++ 2 files changed, 348 insertions(+), 73 deletions(-) diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 677bd91f98..7c65cad94d 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -79,25 +79,25 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): aws_bedrock_runtime_endpoint = optional_params.get( "aws_bedrock_runtime_endpoint", None ) - + # Extract ARN from model string agent_runtime_arn = self._get_agent_runtime_arn(model) - + # Parse ARN to get region region = self._extract_region_from_arn(agent_runtime_arn) - + # Build the base endpoint URL for AgentCore # Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure if aws_bedrock_runtime_endpoint: base_url = aws_bedrock_runtime_endpoint else: base_url = f"https://bedrock-agentcore.{region}.amazonaws.com" - + # Based on boto3 client.invoke_agent_runtime, the path is: # /runtimes/{URL-ENCODED-ARN}/invocations?qualifier= - encoded_arn = quote(agent_runtime_arn, safe='') + encoded_arn = quote(agent_runtime_arn, safe="") endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations" - + # Add qualifier as query parameter if provided if "qualifier" in optional_params: endpoint_url = f"{endpoint_url}?qualifier={optional_params['qualifier']}" @@ -115,6 +115,19 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): stream: Optional[bool] = None, fake_stream: Optional[bool] = None, ) -> Tuple[dict, Optional[bytes]]: + # Check if api_key (bearer token) is provided for Cognito authentication + jwt_token = optional_params.get("api_key") + if jwt_token: + verbose_logger.debug( + f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..." + ) + headers["Content-Type"] = "application/json" + headers["Authorization"] = f"Bearer {jwt_token}" + # Return headers with bearer token and JSON-encoded body (not SigV4 signed) + return headers, json.dumps(request_data).encode() + + # Otherwise, use AWS SigV4 authentication + verbose_logger.debug("AgentCore: Using AWS SigV4 authentication (IAM)") return self._sign_request( service_name="bedrock-agentcore", headers=headers, @@ -157,16 +170,22 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ session_id = optional_params.get("runtimeSessionId", None) if session_id: + verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}") return session_id # Generate a session ID with 33+ characters - return f"litellm-session-{str(uuid.uuid4())}" + generated_id = f"litellm-session-{str(uuid.uuid4())}" + verbose_logger.debug(f"Generated new session ID: {generated_id}") + return generated_id def _get_runtime_user_id(self, optional_params: dict) -> Optional[str]: """ Get runtime user ID if provided """ - return optional_params.get("runtimeUserId", None) + user_id = optional_params.get("runtimeUserId", None) + if user_id: + verbose_logger.debug(f"Using provided runtimeUserId: {user_id}") + return user_id def transform_request( self, @@ -188,6 +207,10 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): Returns: dict: Payload dict containing the prompt """ + verbose_logger.debug( + f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}" + ) + # Use the last message content as the prompt prompt = convert_content_list_to_str(messages[-1]) @@ -206,17 +229,18 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # The request data is the payload dict (will be JSON encoded by the HTTP handler) # Qualifier will be handled as a query parameter in get_complete_url + verbose_logger.debug(f"PAYLOAD: {payload}") return payload def _extract_sse_json(self, line: str) -> Optional[Dict]: """Extract and parse JSON from an SSE data line.""" - if not line.startswith('data:'): + if not line.startswith("data:"): return None - + json_str = line[5:].strip() if not json_str: return None - + try: data = json.loads(json_str) # Skip non-dict data (some lines contain JSON strings) @@ -230,11 +254,11 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): event_payload = event_data.get("event") if not event_payload: return None - + metadata = event_payload.get("metadata") if metadata and "usage" in metadata: return metadata["usage"] # type: ignore - + return None def _extract_content_delta(self, event_data: Dict) -> Optional[str]: @@ -242,11 +266,11 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): event_payload = event_data.get("event") if not event_payload: return None - + content_block_delta = event_payload.get("contentBlockDelta") if not content_block_delta: return None - + delta = content_block_delta.get("delta", {}) return delta.get("text") @@ -258,7 +282,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): content_list = message.get("content", []) if not isinstance(content_list, list): return "" - + return "".join( block["text"] for block in content_list @@ -270,31 +294,28 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) -> Optional[Usage]: """ Calculate token usage using LiteLLM's token counter. - + Args: model: The model name messages: Input messages content: Response content - + Returns: Usage object with calculated tokens, or None if calculation fails """ try: from litellm.utils import token_counter - + prompt_tokens = token_counter(model=model, messages=messages) completion_tokens = token_counter( - model=model, - text=content, - count_response_tokens=True + model=model, text=content, count_response_tokens=True ) total_tokens = prompt_tokens + completion_tokens - + verbose_logger.debug( - f"Calculated usage - prompt: {prompt_tokens}, " - f"completion: {completion_tokens}, total: {total_tokens}" + f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" ) - + return Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, @@ -307,7 +328,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: """ Parse direct JSON response (non-streaming). - + JSON response structure: { "result": { @@ -317,15 +338,15 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): } """ result = response_json.get("result", {}) - + # Extract content using the same helper as SSE parsing content = self._extract_content_from_message(result) # type: ignore - + # JSON responses don't include usage data return AgentCoreParsedResponse( content=content, usage=None, - final_message=result # type: ignore + final_message=result, # type: ignore ) def _get_parsed_response( @@ -333,16 +354,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) -> AgentCoreParsedResponse: """ Parse AgentCore response based on content type. - + Args: raw_response: Raw HTTP response from AgentCore - + Returns: AgentCoreParsedResponse: Parsed response data """ content_type = raw_response.headers.get("content-type", "").lower() verbose_logger.debug(f"AgentCore response Content-Type: {content_type}") - + # Parse response based on content type if "application/json" in content_type: # Direct JSON response @@ -354,64 +375,66 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # SSE stream response (text/event-stream or default) verbose_logger.debug("Parsing SSE stream response") response_text = raw_response.text - verbose_logger.debug(f"AgentCore response (first 500 chars): {response_text[:500]}") + verbose_logger.debug( + f"AgentCore response (first 500 chars): {response_text[:500]}" + ) return self._parse_sse_stream(response_text) def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse: """ Parse Server-Sent Events (SSE) stream format. Each line starts with 'data:' followed by JSON. - + Returns: AgentCoreParsedResponse: Parsed response with content, usage, and message """ final_message: Optional[AgentCoreMessage] = None usage_data: Optional[AgentCoreUsage] = None content_blocks: List[str] = [] - - for line in response_text.strip().split('\n'): + + for line in response_text.strip().split("\n"): line = line.strip() if not line: continue - + data = self._extract_sse_json(line) if not data: continue - + verbose_logger.debug(f"SSE event keys: {list(data.keys())}") - + # Check for final complete message if "message" in data and isinstance(data["message"], dict): final_message = data["message"] # type: ignore verbose_logger.debug("Found final message") - + # Process event data if "event" in data and isinstance(data["event"], dict): event_payload = data["event"] - verbose_logger.debug(f"Event payload keys: {list(event_payload.keys())}") - + verbose_logger.debug( + f"Event payload keys: {list(event_payload.keys())}" + ) + # Extract usage metadata if usage := self._extract_usage_from_event(data): usage_data = usage verbose_logger.debug(f"Found usage data: {usage_data}") - + # Collect content deltas if text := self._extract_content_delta(data): content_blocks.append(text) - + # Build final content content = ( self._extract_content_from_message(final_message) if final_message else "".join(content_blocks) ) - + verbose_logger.debug(f"Final usage_data: {usage_data}") - + return AgentCoreParsedResponse( - content=content, - usage=usage_data, - final_message=final_message + content=content, usage=usage_data, final_message=final_message ) def get_streaming_response( @@ -421,11 +444,11 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) -> AgentCoreSSEStreamIterator: """ Return a streaming iterator for SSE responses. - + Args: model: The model name raw_response: Raw HTTP response with streaming data - + Returns: AgentCoreSSEStreamIterator: Iterator that yields ModelResponse chunks """ @@ -446,7 +469,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) -> CustomStreamWrapper: """ Get a CustomStreamWrapper for synchronous streaming. - + This is called when stream=True is passed to completion(). """ from litellm.llms.custom_httpx.http_handler import ( @@ -454,10 +477,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): _get_httpx_client, ) from litellm.utils import CustomStreamWrapper - + if client is None or not isinstance(client, HTTPHandler): client = _get_httpx_client(params={}) - + + verbose_logger.debug(f"Making sync streaming request to: {api_base}") + # Make streaming request response = client.post( api_base, @@ -466,22 +491,24 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): stream=True, # THIS IS KEY - tells httpx to not buffer logging_obj=logging_obj, ) - + if response.status_code != 200: raise BedrockError( status_code=response.status_code, message=str(response.read()) ) - + # Create iterator for SSE stream - completion_stream = self.get_streaming_response(model=model, raw_response=response) - + completion_stream = self.get_streaming_response( + model=model, raw_response=response + ) + streaming_response = CustomStreamWrapper( completion_stream=completion_stream, model=model, custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - + # LOGGING logging_obj.post_call( input=messages, @@ -489,7 +516,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): original_response="first stream response received", additional_args={"complete_input_dict": data}, ) - + return streaming_response async def get_async_custom_stream_wrapper( @@ -517,7 +544,11 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): from litellm.utils import CustomStreamWrapper if client is None or not isinstance(client, AsyncHTTPHandler): - client = get_async_httpx_client(llm_provider=cast(Any, "bedrock"), params={}) + client = get_async_httpx_client( + llm_provider=cast(Any, "bedrock"), params={} + ) + + verbose_logger.debug(f"Making async streaming request to: {api_base}") # Make async streaming request response = await client.post( @@ -534,7 +565,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): ) # Create iterator for SSE stream - completion_stream = self.get_streaming_response(model=model, raw_response=response) + completion_stream = self.get_streaming_response( + model=model, raw_response=response + ) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, @@ -583,29 +616,29 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): """ Transform the AgentCore response to LiteLLM ModelResponse format. AgentCore can return either JSON or SSE (Server-Sent Events) stream responses. - + Note: For streaming responses, use get_streaming_response() instead. """ try: # Parse the response based on content type (JSON or SSE) parsed_data = self._get_parsed_response(raw_response) - + content = parsed_data["content"] usage_data = parsed_data["usage"] - + verbose_logger.debug(f"Parsed content length: {len(content)}") verbose_logger.debug(f"Usage data: {usage_data}") - + # Create the message message = Message(content=content, role="assistant") - + # Create choices choice = Choices(finish_reason="stop", index=0, message=message) - + # Update model response model_response.choices = [choice] model_response.model = model - + # Add usage information if available # Note: AgentCore JSON responses don't include usage data # SSE responses may include usage in metadata events @@ -618,11 +651,13 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): setattr(model_response, "usage", usage) else: # Calculate token usage using LiteLLM's token counter - verbose_logger.debug("No usage data from AgentCore - calculating tokens") + verbose_logger.debug( + "No usage data from AgentCore - calculating tokens" + ) calculated_usage = self._calculate_usage(model, messages, content) if calculated_usage: setattr(model_response, "usage", calculated_usage) - + return model_response except Exception as e: @@ -658,4 +693,3 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): custom_llm_provider: Optional[str] = None, ) -> bool: return True - diff --git a/tests/llm_translation/test_bedrock_agentcore.py b/tests/llm_translation/test_bedrock_agentcore.py index 6211fdbcdb..6dc6215a5e 100644 --- a/tests/llm_translation/test_bedrock_agentcore.py +++ b/tests/llm_translation/test_bedrock_agentcore.py @@ -126,3 +126,244 @@ def test_bedrock_agentcore_with_custom_params(): assert "prompt" in request_data assert request_data["prompt"] == "Explain machine learning in simple terms" + +def test_bedrock_agentcore_with_runtime_user_id(): + """ + Test AgentCore with runtimeUserId parameter + """ + import json + + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + messages=[ + { + "role": "user", + "content": "Hello", + } + ], + runtimeUserId="test-user-123", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + print(f"mock_post.call_args.kwargs: {call_kwargs}") + + # Verify headers - user ID should be in header + assert "headers" in call_kwargs + headers = call_kwargs["headers"] + print(f"Headers: {headers}") + assert "X-Amzn-Bedrock-AgentCore-Runtime-User-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] == "test-user-123" + + +def test_bedrock_agentcore_with_session_and_user(): + """ + Test AgentCore with both runtimeSessionId and runtimeUserId + """ + import json + + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + messages=[ + { + "role": "user", + "content": "Test message", + } + ], + runtimeSessionId="session-abc-123", + runtimeUserId="user-xyz-789", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + print(f"mock_post.call_args.kwargs: {call_kwargs}") + + # Verify headers contain both session and user IDs + assert "headers" in call_kwargs + headers = call_kwargs["headers"] + print(f"Headers: {headers}") + assert "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] == "session-abc-123" + assert "X-Amzn-Bedrock-AgentCore-Runtime-User-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] == "user-xyz-789" + + +def test_bedrock_agentcore_with_api_key_bearer_token(): + """ + Test AgentCore with api_key parameter for JWT/Bearer token authentication + """ + import json + + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + test_jwt_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + messages=[ + { + "role": "user", + "content": "Test JWT authentication", + } + ], + api_key=test_jwt_token, + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + print(f"mock_post.call_args.kwargs: {call_kwargs}") + + # Verify Authorization header with Bearer token + assert "headers" in call_kwargs + headers = call_kwargs["headers"] + print(f"Headers: {headers}") + assert "Authorization" in headers + assert headers["Authorization"] == f"Bearer {test_jwt_token}" + assert headers["Content-Type"] == "application/json" + + # Verify the request body is JSON-encoded (not SigV4 signed) + assert "data" in call_kwargs + request_data = json.loads(call_kwargs["data"]) + print(f"Request data: {json.dumps(request_data, indent=2)}") + assert "prompt" in request_data + assert request_data["prompt"] == "Test JWT authentication" + + +def test_bedrock_agentcore_with_all_parameters(): + """ + Test AgentCore with all parameters: api_key, runtimeSessionId, runtimeUserId + """ + import json + + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + test_jwt_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.signature" + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + messages=[ + { + "role": "user", + "content": "Complete test", + } + ], + api_key=test_jwt_token, + runtimeSessionId="full-test-session-id", + runtimeUserId="full-test-user-id", + qualifier="LATEST", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + print(f"mock_post.call_args.kwargs: {call_kwargs}") + + # Verify URL includes qualifier + assert "url" in call_kwargs + url = call_kwargs["url"] + print(f"URL: {url}") + assert "qualifier=LATEST" in url + + # Verify all headers are present + assert "headers" in call_kwargs + headers = call_kwargs["headers"] + print(f"Headers: {headers}") + + # Check Bearer token authorization + assert "Authorization" in headers + assert headers["Authorization"] == f"Bearer {test_jwt_token}" + + # Check session and user IDs + assert "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] == "full-test-session-id" + assert "X-Amzn-Bedrock-AgentCore-Runtime-User-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] == "full-test-user-id" + + # Verify JSON body + assert "data" in call_kwargs + request_data = json.loads(call_kwargs["data"]) + print(f"Request data: {json.dumps(request_data, indent=2)}") + assert "prompt" in request_data + assert request_data["prompt"] == "Complete test" + + +def test_bedrock_agentcore_without_api_key_uses_sigv4(): + """ + Test that AgentCore uses AWS SigV4 signing when api_key is not provided + """ + import json + + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC", + messages=[ + { + "role": "user", + "content": "Test SigV4", + } + ], + # No api_key provided - should use SigV4 + runtimeSessionId="sigv4-test-session", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + print(f"mock_post.call_args.kwargs: {call_kwargs}") + + # Verify headers - should have AWS SigV4 headers, not Bearer token + assert "headers" in call_kwargs + headers = call_kwargs["headers"] + print(f"Headers: {headers}") + + # Should NOT have Bearer Authorization when using SigV4 + if "Authorization" in headers: + assert not headers["Authorization"].startswith("Bearer ") + # Should have AWS4-HMAC-SHA256 signature + assert "AWS4-HMAC-SHA256" in headers["Authorization"] + + # Session ID should still be present + assert "X-Amzn-Bedrock-AgentCore-Runtime-Session-Id" in headers + assert headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] == "sigv4-test-session" + From ea80510f78e25cab0b930bb01e66243fb92a203a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 14 Nov 2025 00:25:54 +0530 Subject: [PATCH 14/19] [Feat] Day-0, Add gpt-5.1 and gpt-5.1-codex family support (#16598) * Add day 0 support for gpt-5.1 models * Add gpt-5.1-codex day 0 support * update pricing values --- ...odel_prices_and_context_window_backup.json | 173 ++++++++++++++++++ model_prices_and_context_window.json | 173 ++++++++++++++++++ 2 files changed, 346 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aa7774c43e..7b8177ed99 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13849,6 +13849,113 @@ "supports_service_tier": true, "supports_vision": true }, + "gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "gpt-5.1-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "gpt-5.1-chat-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_vision": true + }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -14048,6 +14155,72 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "output_cost_per_token_priority": 3.6e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4eac8335e6..fa36e2d608 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13852,6 +13852,113 @@ "supports_service_tier": true, "supports_vision": true }, + "gpt-5.1": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "gpt-5.1-2025-11-13": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true + }, + "gpt-5.1-chat-latest": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_native_streaming": true, + "supports_parallel_function_calling": false, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": false, + "supports_vision": true + }, "gpt-5-pro": { "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -14051,6 +14158,72 @@ "supports_tool_choice": true, "supports_vision": true }, + "gpt-5.1-codex": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, + "gpt-5.1-codex-mini": { + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_priority": 4.5e-07, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_token": 2e-06, + "output_cost_per_token_priority": 3.6e-06, + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": false, + "supports_tool_choice": true, + "supports_vision": true + }, "gpt-5-mini": { "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_flex": 1.25e-08, From 8e0b66a81425ba1e42c7e33a2b886e9d0ab7ed22 Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Fri, 14 Nov 2025 05:33:54 +0900 Subject: [PATCH 15/19] fix: exclude unauthorized MCP servers from allowed server list (#16551) * fix: exclude unauthorized MCP servers from allowed server list * fix: test after resolving merge conflicts --- .../mcp_server/mcp_server_manager.py | 20 ++- .../proxy/_experimental/mcp_server/server.py | 8 +- tests/mcp_tests/test_mcp_server.py | 162 +++++++++++++++++- 3 files changed, 174 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7cd6546e9c..aefbbc8d4a 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -478,6 +478,12 @@ class MCPServerManager: """ Get the allowed MCP Servers for the user """ + from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view + + # If admin, get all servers + if user_api_key_auth and _user_has_admin_view(user_api_key_auth): + return list(self.get_registry().keys()) + try: allowed_mcp_servers = await MCPRequestHandler.get_allowed_mcp_servers( user_api_key_auth @@ -485,18 +491,14 @@ class MCPServerManager: verbose_logger.debug( f"Allowed MCP Servers for user api key auth: {allowed_mcp_servers}" ) - if len(allowed_mcp_servers) > 0: - return allowed_mcp_servers - else: + if len(allowed_mcp_servers) == 0: verbose_logger.debug( - "No allowed MCP Servers found for user api key auth, returning default registry servers" + "No allowed MCP Servers found for user api key auth." ) - return list(self.get_registry().keys()) + return allowed_mcp_servers except Exception as e: - verbose_logger.warning( - f"Failed to get allowed MCP servers: {str(e)}. Returning default registry servers." - ) - return list(self.get_registry().keys()) + verbose_logger.warning(f"Failed to get allowed MCP servers: {str(e)}.") + return [] async def get_tools_for_server(self, server_id: str) -> List[MCPTool]: """ diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 6f4dafb5fe..ce29d2d32e 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -628,8 +628,10 @@ if MCP_AVAILABLE: ) ## CHECK IF USER IS ALLOWED TO CALL THIS TOOL - allowed_mcp_server_ids = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth, + allowed_mcp_server_ids = ( + await global_mcp_server_manager.get_allowed_mcp_servers( + user_api_key_auth=user_api_key_auth, + ) ) allowed_mcp_servers = global_mcp_server_manager.get_mcp_servers_from_ids( @@ -638,7 +640,7 @@ if MCP_AVAILABLE: allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names( mcp_servers=mcp_servers, - allowed_mcp_servers=allowed_mcp_servers, + allowed_mcp_servers=allowed_mcp_servers ) server_name: Optional[str] diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index fbdada9306..8391e8f77e 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -88,7 +88,15 @@ async def test_mcp_server_manager_https_server(): } ) - tools = await mcp_server_manager.list_tools() + allowed_server_ids = list(mcp_server_manager.get_registry().keys()) + assert allowed_server_ids, "Expected registry to contain the configured server" + + with patch.object( + mcp_server_manager, + "get_allowed_mcp_servers", + new=AsyncMock(return_value=allowed_server_ids), + ): + tools = await mcp_server_manager.list_tools() print("TOOLS FROM MCP SERVER MANAGER== ", tools) # Verify tools were returned and properly prefixed @@ -192,7 +200,15 @@ async def test_mcp_http_transport_list_tools_mock(): ) # Call list_tools - tools = await test_manager.list_tools() + allowed_server_ids = list(test_manager.get_registry().keys()) + assert allowed_server_ids, "Expected registry to contain configured server" + + with patch.object( + test_manager, + "get_allowed_mcp_servers", + new=AsyncMock(return_value=allowed_server_ids), + ): + tools = await test_manager.list_tools() # Assertions assert len(tools) == 2 @@ -1530,8 +1546,16 @@ async def test_mcp_protocol_version_passed_to_client(): } ) - # Call list_tools with a specific protocol version from request - await test_manager.list_tools() + allowed_server_ids = list(test_manager.get_registry().keys()) + assert allowed_server_ids, "Expected registry to contain configured server" + + with patch.object( + test_manager, + "get_allowed_mcp_servers", + new=AsyncMock(return_value=allowed_server_ids), + ): + # Call list_tools with a specific protocol version from request + await test_manager.list_tools() # Verify the client was created with the correct protocol version mock_client.list_tools.assert_called() @@ -2436,3 +2460,133 @@ async def test_mcp_server_manager_with_access_groups_integration(): # Should only get servers user has access to assert len(allowed_servers) >= 0 # At least verify no errors mock_get_allowed.assert_called_once_with(user_auth) + + +@pytest.mark.asyncio +async def test_get_allowed_mcp_servers_returns_registry_for_admin(): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + + test_manager = MCPServerManager() + test_manager.load_servers_from_config( + { + "alpha_server": { + "url": "https://alpha.server/mcp", + "transport": MCPTransport.http, + }, + "beta_server": { + "url": "https://beta.server/mcp", + "transport": MCPTransport.http, + }, + } + ) + + admin_auth = UserAPIKeyAuth( + api_key="admin-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + with patch.object( + MCPRequestHandler, "get_allowed_mcp_servers", new_callable=AsyncMock + ) as mock_permission_lookup: + allowed_servers = await test_manager.get_allowed_mcp_servers(admin_auth) + + assert set(allowed_servers) == set(test_manager.get_registry().keys()) + mock_permission_lookup.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_allowed_mcp_servers_returns_empty_for_non_admin_without_permissions(): + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( + MCPRequestHandler, + ) + + test_manager = MCPServerManager() + test_manager.load_servers_from_config( + { + "alpha_server": { + "url": "https://alpha.server/mcp", + "transport": MCPTransport.http, + }, + "beta_server": { + "url": "https://beta.server/mcp", + "transport": MCPTransport.http, + }, + } + ) + + user_auth = UserAPIKeyAuth( + api_key="user-key", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + with patch.object( + MCPRequestHandler, "get_allowed_mcp_servers", new_callable=AsyncMock + ) as mock_permission_lookup: + mock_permission_lookup.return_value = [] + allowed_servers = await test_manager.get_allowed_mcp_servers(user_auth) + + assert allowed_servers == [] + mock_permission_lookup.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_call_mcp_tool_uses_manager_permission_lookup(): + from litellm.proxy._experimental.mcp_server.server import ( + call_mcp_tool, + global_mcp_server_manager, + ) + + mock_server = MCPServer( + server_id="server-123", + name="test_server", + alias="test_server", + server_name="test_server", + url="https://test-server.com/mcp", + transport=MCPTransport.http, + mcp_info={"server_name": "test_server"}, + ) + + expected_response = [TextContent(type="text", text="ok")] + + with patch.object( + global_mcp_server_manager, + "get_allowed_mcp_servers", + new_callable=AsyncMock, + ) as mock_get_allowed, patch.object( + global_mcp_server_manager, + "get_mcp_servers_from_ids", + return_value=[mock_server], + ), patch.object( + global_mcp_server_manager, + "_get_mcp_server_from_tool_name", + return_value=mock_server, + ) as mock_get_server, patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_tool_registry" + ) as mock_tool_registry, patch( + "litellm.proxy._experimental.mcp_server.server._handle_managed_mcp_tool", + new_callable=AsyncMock, + ) as mock_handle_managed, patch( + "litellm.proxy._experimental.mcp_server.server.MCPRequestHandler.is_tool_allowed", + return_value=True, + ): + mock_get_allowed.return_value = [mock_server.server_id] + mock_tool_registry.get_tool.return_value = None + mock_handle_managed.return_value = expected_response + + result = await call_mcp_tool( + name=f"{mock_server.name}/gmail_send_email", + arguments={"body": "hello"}, + mcp_servers=["test_server"], + ) + + assert result == expected_response + mock_get_allowed.assert_awaited_once() + assert mock_get_server.call_count == 2 + assert ( + mock_get_server.call_args_list[0][0][0] + == f"{mock_server.name}/gmail_send_email" + ) From 713348828213493407dde134c0f93bfc4fecea67 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Nov 2025 12:41:00 -0800 Subject: [PATCH 16/19] [Feat] VertexAI - Add BGE Embeddings support (#16033) * Support for Custom Vertex AI Models via PSC Endpoint with api_base (#15953) * Support for Custom Vertex AI Models via PSC Endpoint with api_base * Add docs related psc * remove not needed files * remove print statemnt * fix mypy errors * add TextEmbeddingBGEInput * add VertexBGEConfig * add BGE handling * test_vertex_ai_bge_embedding_with_custom_api_base * fix request transform vertex BGE * test_vertex_ai_bge_embedding_with_custom_api_base * tes BGE * test_is_bge_model_detection * docs cleanup * handling BGE URL * fix VertexBGEConfig * test_vertex_ai_bge_with_endpoint_id_pattern * docs vertex BGE * docs * docs fix * fix VertexAIModelRoute * from ..common_utils import VertexAIError, get_vertex_base_model_name add * fix VertexAIGemmaModels * fix get_vertex_base_model_name * test_vertex_ai_bge_psc_endpoint_url_construction --------- Co-authored-by: Sameer Kankute --- docs/my-website/docs/providers/vertex.md | 556 ++--------------- .../docs/providers/vertex_embedding.md | 587 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/llms/vertex_ai/batches/handler.py | 8 + litellm/llms/vertex_ai/common_utils.py | 95 ++- .../vertex_ai_context_caching.py | 4 + .../llms/vertex_ai/vertex_embeddings/bge.py | 182 ++++++ .../vertex_embeddings/transformation.py | 21 +- .../llms/vertex_ai/vertex_embeddings/types.py | 8 +- .../vertex_ai/vertex_gemma_models/main.py | 5 +- litellm/llms/vertex_ai/vertex_llm_base.py | 52 +- .../vertex_ai/vertex_model_garden/main.py | 8 +- .../llms/vertex_ai/test_bge_embedding.py | 251 ++++++++ .../test_bge_response_transformation.py | 111 ++++ .../test_vertex_ai_psc_endpoint_support.py | 258 ++++++++ 15 files changed, 1621 insertions(+), 526 deletions(-) create mode 100644 docs/my-website/docs/providers/vertex_embedding.md create mode 100644 litellm/llms/vertex_ai/vertex_embeddings/bge.py create mode 100644 tests/test_litellm/llms/vertex_ai/test_bge_embedding.py create mode 100644 tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py create mode 100644 tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 874b637e4d..4d7e85f388 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1604,6 +1604,53 @@ litellm.vertex_location = "us-central1 # Your Location | gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` | | gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` | +## Private Service Connect (PSC) Endpoints + +LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments. + +### Usage + +```python +from litellm import completion + +# Use PSC endpoint with custom api_base +response = completion( + model="vertex_ai/1234567890", # Numeric endpoint ID + messages=[{"role": "user", "content": "Hello!"}], + api_base="http://10.96.32.8", # Your PSC endpoint + vertex_project="my-project-id", + vertex_location="us-central1" +) +``` + +**Key Features:** +- Supports both numeric endpoint IDs and custom model names +- Works with both completion and embedding endpoints +- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}` +- Compatible with streaming requests + +### Configuration + +Add PSC endpoints to your `config.yaml`: + +```yaml +model_list: + - model_name: psc-gemini + litellm_params: + model: vertex_ai/1234567890 # Numeric endpoint ID + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" + - model_name: psc-embedding + litellm_params: + model: vertex_ai/text-embedding-004 + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" +``` + ## Fine-tuned Models You can call fine-tuned Vertex AI Gemini models through LiteLLM @@ -2042,515 +2089,6 @@ curl http://0.0.0.0:4000/v1/chat/completions \ | code-gecko@latest| `completion('code-gecko@latest', messages)` | -## **Embedding Models** - -#### Usage - Embedding - - - - -```python -import litellm -from litellm import embedding -litellm.vertex_project = "hardy-device-38811" # Your Project ID -litellm.vertex_location = "us-central1" # proj location - -response = embedding( - model="vertex_ai/textembedding-gecko", - input=["good morning from litellm"], -) -print(response) -``` - - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: snowflake-arctic-embed-m-long-1731622468876 - litellm_params: - model: vertex_ai/ - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request using OpenAI Python SDK, Langchain Python SDK - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="snowflake-arctic-embed-m-long-1731622468876", - input = ["good morning from litellm", "this is another item"], -) - -print(response) -``` - - - - - -#### Supported Embedding Models -All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported - -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` | -| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` | -| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | -| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | -| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | -| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | -| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | -| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | -| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | -| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/", input)` | - -### Supported OpenAI (Unified) Params - -| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) | -|-------|-------------|--------------------| -| `input` | **string or List[string]** | `instances` | -| `dimensions` | **int** | `output_dimensionality` | -| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` | - -#### Usage with OpenAI (Unified) Params - - - - - -```python -response = litellm.embedding( - model="vertex_ai/text-embedding-004", - input=["good morning from litellm", "gm"] - input_type = "RETRIEVAL_DOCUMENT", - dimensions=1, -) -``` - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="text-embedding-004", - input = ["good morning from litellm", "gm"], - dimensions=1, - extra_body = { - "input_type": "RETRIEVAL_QUERY", - } -) - -print(response) -``` - - - - -### Supported Vertex Specific Params - -| param | type | -|-------|-------------| -| `auto_truncate` | **bool** | -| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | -| `title` | **str** | - -#### Usage with Vertex Specific Params (Use `task_type` and `title`) - -You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this: - -[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body) - - - - -```python -response = litellm.embedding( - model="vertex_ai/text-embedding-004", - input=["good morning from litellm", "gm"] - task_type = "RETRIEVAL_DOCUMENT", - title = "test", - dimensions=1, - auto_truncate=True, -) -``` - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="text-embedding-004", - input = ["good morning from litellm", "gm"], - dimensions=1, - extra_body = { - "task_type": "RETRIEVAL_QUERY", - "auto_truncate": True, - "title": "test", - } -) - -print(response) -``` - - - -## **Multi-Modal Embeddings** - - -Known Limitations: -- Only supports 1 image / video / image per request -- Only supports GCS or base64 encoded images / videos - -### Usage - - - - -Using GCS Images - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image -) -``` - -Using base 64 encoded images - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image -) -``` - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: multimodalembedding@001 - litellm_params: - model: vertex_ai/multimodalembedding@001 - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK, Langchain Python SDK - - - - - - -Requests with GCS Image / Video URI - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", -) - -print(response) -``` - -Requests with base64 encoded images - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = "data:image/jpeg;base64,...", -) - -print(response) -``` - - - - - -Requests with GCS Image / Video URI -```python -from langchain_openai import OpenAIEmbeddings - -embeddings_models = "multimodalembedding@001" - -embeddings = OpenAIEmbeddings( - model="multimodalembedding@001", - base_url="http://0.0.0.0:4000", - api_key="sk-1234", # type: ignore -) - - -query_result = embeddings.embed_query( - "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" -) -print(query_result) - -``` - -Requests with base64 encoded images - -```python -from langchain_openai import OpenAIEmbeddings - -embeddings_models = "multimodalembedding@001" - -embeddings = OpenAIEmbeddings( - model="multimodalembedding@001", - base_url="http://0.0.0.0:4000", - api_key="sk-1234", # type: ignore -) - - -query_result = embeddings.embed_query( - "data:image/jpeg;base64,..." -) -print(query_result) - -``` - - - - - - - - - -1. Add model to config.yaml -```yaml -default_vertex_config: - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK - -```python -import vertexai - -from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video -from vertexai.vision_models import VideoSegmentConfig -from google.auth.credentials import Credentials - - -LITELLM_PROXY_API_KEY = "sk-1234" -LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai" - -import datetime - -class CredentialsWrapper(Credentials): - def __init__(self, token=None): - super().__init__() - self.token = token - self.expiry = None # or set to a future date if needed - - def refresh(self, request): - pass - - def apply(self, headers, token=None): - headers['Authorization'] = f'Bearer {self.token}' - - @property - def expired(self): - return False # Always consider the token as non-expired - - @property - def valid(self): - return True # Always consider the credentials as valid - -credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) - -vertexai.init( - project="adroit-crow-413218", - location="us-central1", - api_endpoint=LITELLM_PROXY_BASE, - credentials = credentials, - api_transport="rest", - -) - -model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding") -image = Image.load_from_file( - "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" -) - -embeddings = model.get_embeddings( - image=image, - contextual_text="Colosseum", - dimension=1408, -) -print(f"Image Embedding: {embeddings.image_embedding}") -print(f"Text Embedding: {embeddings.text_embedding}") -``` - - - - - -### Text + Image + Video Embeddings - - - - -Text + Image - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image -) -``` - -Text + Video - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image -) -``` - -Image + Video - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image -) -``` - - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: multimodalembedding@001 - litellm_params: - model: vertex_ai/multimodalembedding@001 - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK, Langchain Python SDK - - -Text + Image - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"], -) - -print(response) -``` - -Text + Video -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"], -) - -print(response) -``` - -Image + Video -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"], -) - -print(response) -``` - - - - - ## **Gemini TTS (Text-to-Speech) Audio Output** :::info diff --git a/docs/my-website/docs/providers/vertex_embedding.md b/docs/my-website/docs/providers/vertex_embedding.md new file mode 100644 index 0000000000..5656ade337 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_embedding.md @@ -0,0 +1,587 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Embedding + +## Usage - Embedding + + + + +```python +import litellm +from litellm import embedding +litellm.vertex_project = "hardy-device-38811" # Your Project ID +litellm.vertex_location = "us-central1" # proj location + +response = embedding( + model="vertex_ai/textembedding-gecko", + input=["good morning from litellm"], +) +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: snowflake-arctic-embed-m-long-1731622468876 + litellm_params: + model: vertex_ai/ + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK, Langchain Python SDK + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="snowflake-arctic-embed-m-long-1731622468876", + input = ["good morning from litellm", "this is another item"], +) + +print(response) +``` + + + + + +#### Supported Embedding Models +All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` | +| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` | +| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | +| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | +| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | +| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | +| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | +| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | +| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | +| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/", input)` | + +### Supported OpenAI (Unified) Params + +| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) | +|-------|-------------|--------------------| +| `input` | **string or List[string]** | `instances` | +| `dimensions` | **int** | `output_dimensionality` | +| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` | + +#### Usage with OpenAI (Unified) Params + + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + input_type = "RETRIEVAL_DOCUMENT", + dimensions=1, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "input_type": "RETRIEVAL_QUERY", + } +) + +print(response) +``` + + + + +### Supported Vertex Specific Params + +| param | type | +|-------|-------------| +| `auto_truncate` | **bool** | +| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | +| `title` | **str** | + +#### Usage with Vertex Specific Params (Use `task_type` and `title`) + +You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this: + +[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body) + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + task_type = "RETRIEVAL_DOCUMENT", + title = "test", + dimensions=1, + auto_truncate=True, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "task_type": "RETRIEVAL_QUERY", + "auto_truncate": True, + "title": "test", + } +) + +print(response) +``` + + + +## **BGE Embeddings** + +Use BGE (Baidu General Embedding) models deployed on Vertex AI. + +### Usage + + + + +```python showLineNumbers title="Using BGE on Vertex AI" +import litellm + +response = litellm.embedding( + model="vertex_ai/bge/", + input=["Hello", "World"], + vertex_project="your-project-id", + vertex_location="your-location" +) + +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: bge-embedding + litellm_params: + model: vertex_ai/bge/ + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: your-credentials.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +```bash +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK + +```python showLineNumbers title="Making requests to BGE" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="bge-embedding", + input=["good morning from litellm", "this is another item"] +) + +print(response) +``` + +Using a Private Service Connect (PSC) endpoint + +```yaml showLineNumbers title="config.yaml (PSC)" +model_list: + - model_name: bge-small-en-v1.5 + litellm_params: + model: vertex_ai/bge/1234567890 + api_base: http://10.96.32.8 # Your PSC IP + vertex_project: my-project-id #optional + vertex_location: us-central1 #optional +``` + + + + +## **Multi-Modal Embeddings** + + +Known Limitations: +- Only supports 1 image / video / image per request +- Only supports GCS or base64 encoded images / videos + +### Usage + + + + +Using GCS Images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image +) +``` + +Using base 64 encoded images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image +) +``` + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + + + + + +Requests with GCS Image / Video URI + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", +) + +print(response) +``` + +Requests with base64 encoded images + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "data:image/jpeg;base64,...", +) + +print(response) +``` + + + + + +Requests with GCS Image / Video URI +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) +print(query_result) + +``` + +Requests with base64 encoded images + +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "data:image/jpeg;base64,..." +) +print(query_result) + +``` + + + + + + + + + +1. Add model to config.yaml +```yaml +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK + +```python +import vertexai + +from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video +from vertexai.vision_models import VideoSegmentConfig +from google.auth.credentials import Credentials + + +LITELLM_PROXY_API_KEY = "sk-1234" +LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai" + +import datetime + +class CredentialsWrapper(Credentials): + def __init__(self, token=None): + super().__init__() + self.token = token + self.expiry = None # or set to a future date if needed + + def refresh(self, request): + pass + + def apply(self, headers, token=None): + headers['Authorization'] = f'Bearer {self.token}' + + @property + def expired(self): + return False # Always consider the token as non-expired + + @property + def valid(self): + return True # Always consider the credentials as valid + +credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) + +vertexai.init( + project="adroit-crow-413218", + location="us-central1", + api_endpoint=LITELLM_PROXY_BASE, + credentials = credentials, + api_transport="rest", + +) + +model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding") +image = Image.load_from_file( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) + +embeddings = model.get_embeddings( + image=image, + contextual_text="Colosseum", + dimension=1408, +) +print(f"Image Embedding: {embeddings.image_embedding}") +print(f"Text Embedding: {embeddings.text_embedding}") +``` + + + + + +### Text + Image + Video Embeddings + + + + +Text + Image + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image +) +``` + +Text + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + +Image + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + +Text + Image + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"], +) + +print(response) +``` + +Text + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + +Image + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + + + \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 0b6c31cc1f..809e0bc0b7 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -488,6 +488,7 @@ const sidebars = { "providers/vertex_ai/videos", "providers/vertex_partner", "providers/vertex_self_deployed", + "providers/vertex_embedding", "providers/vertex_image", "providers/vertex_batch", "providers/vertex_ocr", diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 7932881f48..b40f0a72a5 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -61,6 +61,10 @@ class VertexAIBatchPrediction(VertexLLM): stream=None, auth_header=None, url=default_api_base, + model=None, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1", ) headers = { @@ -166,6 +170,10 @@ class VertexAIBatchPrediction(VertexLLM): stream=None, auth_header=None, url=default_api_base, + model=None, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1", ) headers = { diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 2c53457736..02b0f79280 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -31,9 +31,11 @@ class VertexAIModelRoute(str, Enum): PARTNER_MODELS = "partner_models" GEMINI = "gemini" GEMMA = "gemma" + BGE = "bge" MODEL_GARDEN = "model_garden" NON_GEMINI = "non_gemini" +VERTEX_AI_MODEL_ROUTES = [f"{route.value}/" for route in VertexAIModelRoute] def get_vertex_ai_model_route( model: str, litellm_params: Optional[dict] = None @@ -60,6 +62,9 @@ def get_vertex_ai_model_route( >>> get_vertex_ai_model_route("openai/gpt-oss-120b") VertexAIModelRoute.MODEL_GARDEN + + >>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"}) + VertexAIModelRoute.GEMINI # Numeric endpoints with api_base use HTTP path """ from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( VertexAIPartnerModels, @@ -69,11 +74,20 @@ def get_vertex_ai_model_route( if litellm_params and litellm_params.get("base_model") is not None: if "gemini" in litellm_params["base_model"]: return VertexAIModelRoute.GEMINI - + + # Check if numeric endpoint ID with custom api_base (PSC endpoint) + # Route to GEMINI (HTTP path) to support PSC endpoints properly + if model.isdigit() and litellm_params and litellm_params.get("api_base"): + return VertexAIModelRoute.GEMINI + # Check for partner models (llama, mistral, claude, etc.) if VertexAIPartnerModels.is_vertex_partner_model(model=model): return VertexAIModelRoute.PARTNER_MODELS - + + # Check for BGE models + if "bge/" in model or "bge" in model.lower(): + return VertexAIModelRoute.BGE + # Check for gemma models if "gemma/" in model: return VertexAIModelRoute.GEMMA @@ -136,6 +150,71 @@ all_gemini_url_modes = Literal[ ] +def get_vertex_base_model_name(model: str) -> str: + """ + Strip routing prefixes from model name for PSC/endpoint URL construction. + + Patterns like "bge/", "gemma/", "openai/" are used for internal routing but + should not appear in the actual endpoint URL. Routing prefixes are derived + from VertexAIModelRoute enum values. + + Args: + model: The model name with potential prefix (e.g., "bge/123456", "gemma/gemma-3-12b-it") + + Returns: + str: The model name without routing prefix (e.g., "123456", "gemma-3-12b-it") + + Examples: + >>> get_vertex_base_model_name("bge/378943383978115072") + "378943383978115072" + + >>> get_vertex_base_model_name("gemma/gemma-3-12b-it") + "gemma-3-12b-it" + + >>> get_vertex_base_model_name("openai/gpt-oss-120b") + "gpt-oss-120b" + + >>> get_vertex_base_model_name("1234567890") + "1234567890" + """ + # Derive routing prefixes from VertexAIModelRoute enum + # Map specific routes to their prefixes (some routes like PARTNER_MODELS, GEMINI don't have prefixes) + + + for route in VERTEX_AI_MODEL_ROUTES: + if model.startswith(route): + return model.replace(route, "", 1) + + return model + + +def _get_embedding_url( + model: str, + vertex_project: Optional[str], + vertex_location: Optional[str], + vertex_api_version: Literal["v1", "v1beta1"], +) -> Tuple[str, str]: + """ + Get URL for embedding models. + + Handles special patterns: + - bge/endpoint_id -> strips to endpoint_id for endpoints/ routing + - numeric model -> routes to endpoints/ + - regular model -> routes to publishers/google/models/ + """ + endpoint = "predict" + + # Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction + model = get_vertex_base_model_name(model=model) + + url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + if model.isdigit(): + # https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict + url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + + return url, endpoint + + def _get_vertex_url( mode: all_gemini_url_modes, model: str, @@ -148,6 +227,7 @@ def _get_vertex_url( endpoint: Optional[str] = None model = litellm.VertexGeminiConfig.get_model_for_vertex_ai_url(model=model) + if mode == "chat": ### SET RUNTIME ENDPOINT ### endpoint = "generateContent" @@ -172,11 +252,12 @@ def _get_vertex_url( if stream is True: url += "?alt=sse" elif mode == "embedding": - endpoint = "predict" - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" - if model.isdigit(): - # https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + return _get_embedding_url( + model=model, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_api_version=vertex_api_version, + ) elif mode == "image_generation": endpoint = "predict" url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 70b068b5a4..dabc620a6d 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -79,6 +79,10 @@ class ContextCachingEndpoints(VertexBase): stream=None, auth_header=auth_header, url=url, + model=None, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_api_version="v1beta1" if custom_llm_provider == "vertex_ai_beta" else "v1", ) def check_cache( diff --git a/litellm/llms/vertex_ai/vertex_embeddings/bge.py b/litellm/llms/vertex_ai/vertex_embeddings/bge.py new file mode 100644 index 0000000000..2eff0ba96d --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_embeddings/bge.py @@ -0,0 +1,182 @@ +""" +Vertex AI BGE (BAAI General Embedding) Configuration + +BGE models deployed on Vertex AI require different input/output format: +- Request: Use "prompt" instead of "content" as the input field +- Response: Embeddings are returned directly as arrays, not wrapped in objects + +Model name handling: +- Model names like "bge/endpoint_id" are automatically transformed in common_utils._get_vertex_url() +- This module focuses on request/response transformation only +""" + +from typing import List, Optional, Union + +from litellm.types.utils import EmbeddingResponse, Usage + +from .types import ( + EmbeddingParameters, + TaskType, + TextEmbeddingBGEInput, + VertexEmbeddingRequest, +) + + +class VertexBGEConfig: + """ + Configuration and transformation logic for BGE models on Vertex AI. + + BGE (BAAI General Embedding) models use a different request format + where the input field is named "prompt" instead of "content". + + Supported model patterns (after provider split in main.py): + - "bge-small-en-v1.5" (model name) + - "bge/204379420394258432" (endpoint ID pattern) + + Note: Model name transformation (bge/ -> numeric ID) is handled automatically + in common_utils._get_vertex_url(). This class focuses on request/response format only. + """ + + @staticmethod + def is_bge_model(model: str) -> bool: + """ + Check if the model is a BGE (BAAI General Embedding) model. + + After provider split in main.py, supports: + - "bge-small-en-v1.5" (model name) + - "bge/204379420394258432" (endpoint ID pattern) + + Args: + model: The model name after provider split + + Returns: + bool: True if the model is a BGE model + """ + model_lower = model.lower() + # Check for "bge/" prefix (endpoint pattern) or "bge" in model name + return model_lower.startswith("bge/") or "bge" in model_lower + + @staticmethod + def transform_request( + input: Union[list, str], optional_params: dict, model: str + ) -> VertexEmbeddingRequest: + """ + Transforms an OpenAI request to a Vertex BGE embedding request. + + BGE models use "prompt" instead of "content" as the input field. + + Args: + input: The input text(s) to embed + optional_params: Optional parameters for the request + model: The model name + + Returns: + VertexEmbeddingRequest: The transformed request + """ + vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest() + vertex_text_embedding_input_list: List[TextEmbeddingBGEInput] = [] + task_type: Optional[TaskType] = optional_params.get("task_type") + title = optional_params.get("title") + + if isinstance(input, str): + input = [input] + + for text in input: + embedding_input = VertexBGEConfig._create_embedding_input( + prompt=text, task_type=task_type, title=title + ) + vertex_text_embedding_input_list.append(embedding_input) + + vertex_request["instances"] = vertex_text_embedding_input_list + vertex_request["parameters"] = EmbeddingParameters(**optional_params) + + return vertex_request + + @staticmethod + def _create_embedding_input( + prompt: str, + task_type: Optional[TaskType] = None, + title: Optional[str] = None, + ) -> TextEmbeddingBGEInput: + """ + Creates a TextEmbeddingBGEInput object for BGE models. + + BGE models use "prompt" instead of "content" as the input field. + + Args: + prompt: The prompt to be embedded + task_type: The type of task to be performed + title: The title of the document to be embedded + + Returns: + TextEmbeddingBGEInput: A TextEmbeddingBGEInput object + """ + text_embedding_input = TextEmbeddingBGEInput(prompt=prompt) + if task_type is not None: + text_embedding_input["task_type"] = task_type + if title is not None: + text_embedding_input["title"] = title + return text_embedding_input + + @staticmethod + def transform_response( + response: dict, model: str, model_response: EmbeddingResponse + ) -> EmbeddingResponse: + """ + Transforms a Vertex BGE embedding response to OpenAI format. + + BGE models return embeddings directly as arrays in predictions: + { + "predictions": [ + [0.002, 0.021, ...], + [0.003, 0.022, ...] + ] + } + + Args: + response: The raw response from Vertex AI + model: The model name + model_response: The EmbeddingResponse object to populate + + Returns: + EmbeddingResponse: The transformed response in OpenAI format + + Raises: + KeyError: If response doesn't contain 'predictions' + ValueError: If predictions is not a list or contains invalid data + """ + if "predictions" not in response: + raise KeyError("Response missing 'predictions' field") + + _predictions = response["predictions"] + + if not isinstance(_predictions, list): + raise ValueError(f"Expected 'predictions' to be a list, got {type(_predictions)}") + + embedding_response = [] + # BGE models don't return token counts, so we estimate or set to 0 + input_tokens = 0 + + for idx, embedding_values in enumerate(_predictions): + if not isinstance(embedding_values, list): + raise ValueError( + f"Expected embedding at index {idx} to be a list, got {type(embedding_values)}" + ) + + embedding_response.append( + { + "object": "embedding", + "index": idx, + "embedding": embedding_values, + } + ) + + model_response.object = "list" + model_response.data = embedding_response + model_response.model = model + usage = Usage( + prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens + ) + setattr(model_response, "usage", usage) + return model_response + diff --git a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py index 97af558041..5a3a4a7188 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/transformation.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/transformation.py @@ -105,10 +105,16 @@ class VertexAITextEmbeddingConfig(BaseModel): """ Transforms an openai request to a vertex embedding request. """ + # Import here to avoid circular import issues with litellm.__init__ + from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig if model.isdigit(): return self._transform_openai_request_to_fine_tuned_embedding_request( input, optional_params, model ) + if VertexBGEConfig.is_bge_model(model): + return VertexBGEConfig.transform_request( + input=input, optional_params=optional_params, model=model + ) vertex_request: VertexEmbeddingRequest = VertexEmbeddingRequest() vertex_text_embedding_input_list: List[TextEmbeddingInput] = [] @@ -167,6 +173,9 @@ class VertexAITextEmbeddingConfig(BaseModel): vertex_request["parameters"] = TextEmbeddingFineTunedParameters( **optional_params ) + # Remove 'shared_session' from parameters if present + if vertex_request["parameters"] is not None and "shared_session" in vertex_request["parameters"]: + del vertex_request["parameters"]["shared_session"] # type: ignore[typeddict-item] return vertex_request @@ -183,8 +192,8 @@ class VertexAITextEmbeddingConfig(BaseModel): Args: content (str): The content to be embedded. - task_type (Optional[TaskType]): The type of task to be performed". - title (Optional[str]): The title of the document to be embedded + task_type (Optional[TaskType]): The type of task to be performed. + title (Optional[str]): The title of the document to be embedded. Returns: TextEmbeddingInput: A TextEmbeddingInput object. @@ -206,6 +215,14 @@ class VertexAITextEmbeddingConfig(BaseModel): return self._transform_vertex_response_to_openai_for_fine_tuned_models( response, model, model_response ) + + # Import here to avoid circular import issues with litellm.__init__ + from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig + + if VertexBGEConfig.is_bge_model(model): + return VertexBGEConfig.transform_response( + response=response, model=model, model_response=model_response + ) _predictions = response["predictions"] diff --git a/litellm/llms/vertex_ai/vertex_embeddings/types.py b/litellm/llms/vertex_ai/vertex_embeddings/types.py index 7f85ea46f3..fa9794d79a 100644 --- a/litellm/llms/vertex_ai/vertex_embeddings/types.py +++ b/litellm/llms/vertex_ai/vertex_embeddings/types.py @@ -25,6 +25,12 @@ class TextEmbeddingInput(TypedDict, total=False): title: Optional[str] +class TextEmbeddingBGEInput(TypedDict, total=False): + prompt: str + task_type: Optional[TaskType] + title: Optional[str] + + # Fine-tuned models require a different input format # Ref: https://console.cloud.google.com/vertex-ai/model-garden?hl=en&project=adroit-crow-413218&pageState=(%22galleryStateKey%22:(%22f%22:(%22g%22:%5B%5D,%22o%22:%5B%5D),%22s%22:%22%22)) class TextEmbeddingFineTunedInput(TypedDict, total=False): @@ -44,7 +50,7 @@ class EmbeddingParameters(TypedDict, total=False): class VertexEmbeddingRequest(TypedDict, total=False): - instances: Union[List[TextEmbeddingInput], List[TextEmbeddingFineTunedInput]] + instances: Union[List[TextEmbeddingInput], List[TextEmbeddingBGEInput], List[TextEmbeddingFineTunedInput]] parameters: Optional[Union[EmbeddingParameters, TextEmbeddingFineTunedParameters]] diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/main.py b/litellm/llms/vertex_ai/vertex_gemma_models/main.py index 8203b285eb..41bd6b5431 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/main.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/main.py @@ -25,7 +25,7 @@ import httpx # type: ignore from litellm.utils import ModelResponse -from ..common_utils import VertexAIError +from ..common_utils import VertexAIError, get_vertex_base_model_name from ..vertex_llm_base import VertexBase @@ -82,7 +82,8 @@ class VertexAIGemmaModels(VertexBase): message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""", ) try: - model = model.replace("gemma/", "") + + model = get_vertex_base_model_name(model=model) vertex_httpx_logic = VertexLLM() access_token, project_id = vertex_httpx_logic._ensure_access_token( diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 9ddbc461a7..ce50bf311e 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -19,6 +19,7 @@ from .common_utils import ( _get_gemini_url, _get_vertex_url, all_gemini_url_modes, + get_vertex_base_model_name, is_global_only_vertex_model, ) @@ -241,6 +242,9 @@ class VertexBase: auth_header=None, url=default_api_base, model=model, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1", # Partner models typically use v1 ) return api_base @@ -289,9 +293,18 @@ class VertexBase: auth_header: Optional[str], url: str, model: Optional[str] = None, + vertex_project: Optional[str] = None, + vertex_location: Optional[str] = None, + vertex_api_version: Optional[Literal["v1", "v1beta1"]] = None, ) -> Tuple[Optional[str], str]: """ for cloudflare ai gateway - https://github.com/BerriAI/litellm/issues/4317 + + Handles custom api_base for: + 1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint} + 2. Vertex AI with standard proxies - constructs {api_base}:{endpoint} + 3. Vertex AI with PSC endpoints - constructs full path structure + {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} ## Returns - (auth_header, url) - Tuple[Optional[str], str] @@ -311,8 +324,37 @@ class VertexBase: if gemini_api_key is not None: auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment] else: - url = "{}:{}".format(api_base, endpoint) - + # For Vertex AI + # Check if this is a PSC endpoint or custom deployment + # PSC/custom endpoints need the full path structure + if vertex_project and vertex_location and model: + # Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction + model_for_url = get_vertex_base_model_name(model=model) + + # Check if model is numeric (endpoint ID) or if api_base doesn't contain googleapis.com + # These are indicators of PSC/custom endpoints + is_psc_or_custom = ( + "googleapis.com" not in api_base.lower() or model_for_url.isdigit() + ) + + if is_psc_or_custom: + # Construct full PSC/custom endpoint URL + # Format: {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} + version = vertex_api_version or "v1" + url = "{}/{}/projects/{}/locations/{}/endpoints/{}:{}".format( + api_base.rstrip("/"), + version, + vertex_project, + vertex_location, + model_for_url, + endpoint, + ) + else: + # Standard proxy - just append endpoint + url = "{}:{}".format(api_base, endpoint) + else: + # Fallback to simple format if we don't have all parameters + url = "{}:{}".format(api_base, endpoint) if stream is True: url = url + "?alt=sse" return auth_header, url @@ -339,6 +381,7 @@ class VertexBase: Returns token, url """ + version: Optional[Literal["v1beta1", "v1"]] = None if custom_llm_provider == "gemini": url, endpoint = _get_gemini_url( mode=mode, @@ -354,7 +397,7 @@ class VertexBase: ) ### SET RUNTIME ENDPOINT ### - version: Literal["v1beta1", "v1"] = ( + version = ( "v1beta1" if should_use_v1beta1_features is True else "v1" ) url, endpoint = _get_vertex_url( @@ -375,6 +418,9 @@ class VertexBase: stream=stream, url=url, model=model, + vertex_project=vertex_project, + vertex_location=vertex_location, + vertex_api_version=version, ) def _handle_reauthentication( diff --git a/litellm/llms/vertex_ai/vertex_model_garden/main.py b/litellm/llms/vertex_ai/vertex_model_garden/main.py index 1c57096734..fe7d0862e0 100644 --- a/litellm/llms/vertex_ai/vertex_model_garden/main.py +++ b/litellm/llms/vertex_ai/vertex_model_garden/main.py @@ -22,7 +22,7 @@ import httpx # type: ignore from litellm.utils import ModelResponse -from ..common_utils import VertexAIError +from ..common_utils import VertexAIError, get_vertex_base_model_name from ..vertex_llm_base import VertexBase @@ -89,7 +89,7 @@ class VertexAIModelGardenModels(VertexBase): message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""", ) try: - model = model.replace("openai/", "") + model = get_vertex_base_model_name(model=model) vertex_httpx_logic = VertexLLM() access_token, project_id = vertex_httpx_logic._ensure_access_token( @@ -123,6 +123,10 @@ class VertexAIModelGardenModels(VertexBase): stream=stream, auth_header=None, url=default_api_base, + model=model, + vertex_project=vertex_project or project_id, + vertex_location=vertex_location or "us-central1", + vertex_api_version="v1beta1", ) model = "" return openai_like_chat_completions.completion( diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py b/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py new file mode 100644 index 0000000000..156ab95184 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_bge_embedding.py @@ -0,0 +1,251 @@ +""" +Test BGE embeddings with Vertex AI using custom api_base. + +This test ensures that BGE embeddings work correctly with Vertex AI +and that the request body is properly formatted. +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler + + +def test_vertex_ai_bge_embedding_with_custom_api_base(): + """ + Test Vertex AI BGE embeddings with custom api_base. + + This test verifies that when using a BGE model with Vertex AI and + a custom api_base, the request is properly formatted and sent to + the correct endpoint. + """ + client = HTTPHandler() + + def mock_auth_token(*args, **kwargs): + return "fake-token", "fake-project" + + with patch.object(client, "post") as mock_post, patch( + "litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token", + side_effect=mock_auth_token + ): + mock_response = MagicMock() + mock_response.status_code = 200 + # BGE models return embeddings directly as arrays, not wrapped in objects + mock_response.json.return_value = { + "predictions": [ + [0.1, 0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9, 1.0] + ], + "deployedModelId": "849506872875548672", + "model": "projects/1060139831167/locations/us-central1/models/baai_bge-small-en-v1.5", + "modelDisplayName": "baai_bge-small-en-v1.5", + "modelVersionId": "1" + } + mock_post.return_value = mock_response + + response = litellm.embedding( + model="vertex_ai/bge-small-en-v1.5", + input=["Hello", "World"], + api_base="http://10.96.32.8", + client=client + ) + + mock_post.assert_called_once() + + call_args = mock_post.call_args + kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1] + + if "url" in kwargs: + api_url_called = kwargs["url"] + elif len(call_args[0]) > 0: + api_url_called = call_args[0][0] + else: + api_url_called = "Unknown" + + # Vertex AI may use 'json' or 'data' parameter + if "json" in kwargs: + request_data = kwargs["json"] + elif "data" in kwargs: + request_data = json.loads(kwargs["data"]) + else: + request_data = {} + + print("\n" + "="*50) + print("Mock Request Body Received:") + print("="*50) + print(json.dumps(request_data, indent=2)) + print("="*50) + print(f"API Base: {api_url_called}") + print("="*50 + "\n") + + assert "instances" in request_data + assert len(request_data["instances"]) == 2 + # BGE models should use "prompt" instead of "content" + assert "prompt" in request_data["instances"][0] + assert request_data["instances"][0]["prompt"] == "Hello" + assert "prompt" in request_data["instances"][1] + assert request_data["instances"][1]["prompt"] == "World" + + assert isinstance(response.data, list) + assert len(response.data) == 2 + assert "embedding" in response.data[0] + + +def test_vertex_ai_bge_with_endpoint_id_pattern(): + """ + Test BGE with vertex_ai/bge/endpoint_id pattern. + + This test verifies that the pattern vertex_ai/bge/204379420394258432 + correctly triggers BGE transformations and routes to the endpoint. + """ + client = HTTPHandler() + + def mock_auth_token(*args, **kwargs): + return "fake-token", "fake-project" + + with patch.object(client, "post") as mock_post, patch( + "litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token", + side_effect=mock_auth_token + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + [0.1, 0.2, 0.3, 0.4, 0.5], + [0.6, 0.7, 0.8, 0.9, 1.0] + ], + "deployedModelId": "204379420394258432", + "model": "projects/1060139831167/locations/europe-west4/models/baai_bge-base-en", + "modelDisplayName": "baai_bge-base-en", + "modelVersionId": "1" + } + mock_post.return_value = mock_response + + response = litellm.embedding( + model="vertex_ai/bge/204379420394258432", + input=["Hello", "World"], + vertex_project="1060139831167", + vertex_location="europe-west4", + client=client + ) + + mock_post.assert_called_once() + + call_args = mock_post.call_args + kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1] + + if "url" in kwargs: + api_url_called = kwargs["url"] + elif len(call_args[0]) > 0: + api_url_called = call_args[0][0] + else: + api_url_called = "Unknown" + + # Vertex AI may use 'json' or 'data' parameter + if "json" in kwargs: + request_data = kwargs["json"] + elif "data" in kwargs: + request_data = json.loads(kwargs["data"]) + else: + request_data = {} + + print("\n" + "="*50) + print("BGE Endpoint Pattern Test:") + print("="*50) + print(f"Model: vertex_ai/bge/204379420394258432") + print(f"API URL: {api_url_called}") + print("Request Body:") + print(json.dumps(request_data, indent=2)) + print("="*50 + "\n") + + # Verify URL contains the endpoint ID and uses endpoints/ path + assert "204379420394258432" in api_url_called, f"Endpoint ID not in URL: {api_url_called}" + assert "endpoints" in api_url_called, f"Expected 'endpoints' in URL, got: {api_url_called}" + + # Verify BGE-specific request format (uses "prompt" not "content") + assert "instances" in request_data + assert "prompt" in request_data["instances"][0] + assert request_data["instances"][0]["prompt"] == "Hello" + + # Verify response + assert isinstance(response.data, list) + assert len(response.data) == 2 + + +def test_vertex_ai_bge_psc_endpoint_url_construction(): + """ + Test that BGE models with PSC endpoints construct correct URL without bge/ prefix. + + Verifies that vertex_ai/bge/378943383978115072 with api_base http://10.128.16.2 + constructs URL: http://10.128.16.2/v1/projects/{project}/locations/{location}/endpoints/378943383978115072:predict + + The bge/ prefix should be stripped from the endpoint URL. + """ + client = HTTPHandler() + + def mock_auth_token(*args, **kwargs): + return "fake-token", "gen-lang-client-0682925754" + + with patch.object(client, "post") as mock_post, patch( + "litellm.llms.vertex_ai.vertex_embeddings.embedding_handler.VertexEmbedding._ensure_access_token", + side_effect=mock_auth_token + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "predictions": [ + [0.1, 0.2, 0.3, 0.4, 0.5] + ] + } + mock_post.return_value = mock_response + + response = litellm.embedding( + model="vertex_ai/bge/378943383978115072", + input=["The food was delicious and the waiter.."], + api_base="http://10.128.16.2", + vertex_project="gen-lang-client-0682925754", + vertex_location="us-central1", + client=client + ) + + mock_post.assert_called_once() + + call_args = mock_post.call_args + kwargs = call_args.kwargs if hasattr(call_args, 'kwargs') else call_args[1] + + if "url" in kwargs: + api_url_called = kwargs["url"] + elif len(call_args[0]) > 0: + api_url_called = call_args[0][0] + else: + api_url_called = "Unknown" + + print("\n" + "="*50) + print("PSC Endpoint URL Construction Test:") + print("="*50) + print(f"Model: vertex_ai/bge/378943383978115072") + print(f"API Base: http://10.128.16.2") + print(f"Constructed URL: {api_url_called}") + print("="*50 + "\n") + + # Verify the URL is constructed correctly + expected_url = "http://10.128.16.2/v1/projects/gen-lang-client-0682925754/locations/us-central1/endpoints/378943383978115072:predict" + assert api_url_called == expected_url, f"Expected URL: {expected_url}, Got: {api_url_called}" + + # Verify bge/ prefix is NOT in the URL + assert "bge/" not in api_url_called, f"URL should not contain 'bge/' prefix: {api_url_called}" + + # Verify response works + assert isinstance(response.data, list) + assert len(response.data) == 1 + + diff --git a/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py b/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py new file mode 100644 index 0000000000..20150501ad --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_bge_response_transformation.py @@ -0,0 +1,111 @@ +""" +Test BGE response transformation validation. + +This test verifies that the BGE response transformer properly validates +and handles different response formats. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../../../..") +) + +import pytest + +from litellm.llms.vertex_ai.vertex_embeddings.bge import VertexBGEConfig +from litellm.types.utils import EmbeddingResponse + + +def test_is_bge_model_detection(): + """ + Test BGE model detection for post-provider-split patterns. + + After main.py splits the provider, model strings are passed without the provider prefix. + Model name transformation (bge/ -> numeric ID) is handled in common_utils._get_vertex_url(). + """ + # Should detect BGE models (after provider split) + assert VertexBGEConfig.is_bge_model("bge-small-en-v1.5") is True + assert VertexBGEConfig.is_bge_model("bge/204379420394258432") is True + assert VertexBGEConfig.is_bge_model("BGE-large-en-v1.5") is True # case insensitive + + # Should not detect non-BGE models + assert VertexBGEConfig.is_bge_model("textembedding-gecko") is False + assert VertexBGEConfig.is_bge_model("gemma") is False + assert VertexBGEConfig.is_bge_model("123456789") is False + + +def test_bge_response_transformation_success(): + """ + Test successful BGE response transformation. + + Verifies that a valid BGE response is properly transformed + to OpenAI format. + """ + response = { + "predictions": [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6] + ], + "deployedModelId": "123456", + "model": "projects/test/models/bge-base" + } + + model_response = EmbeddingResponse() + result = VertexBGEConfig.transform_response( + response=response, + model="bge-small-en-v1.5", + model_response=model_response + ) + + assert result.object == "list" + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + assert result.data[0]["index"] == 0 + assert result.data[1]["index"] == 1 + assert result.model == "bge-small-en-v1.5" + + +def test_bge_response_missing_predictions(): + """ + Test BGE response transformation with missing predictions field. + + Verifies that a KeyError is raised when the response doesn't + contain the required 'predictions' field. + """ + response = { + "deployedModelId": "123456", + "model": "projects/test/models/bge-base" + } + + model_response = EmbeddingResponse() + + with pytest.raises(KeyError, match="Response missing 'predictions' field"): + VertexBGEConfig.transform_response( + response=response, + model="bge-small-en-v1.5", + model_response=model_response + ) + + +def test_bge_response_invalid_predictions_type(): + """ + Test BGE response transformation with invalid predictions type. + + Verifies that a ValueError is raised when predictions is not a list. + """ + response = { + "predictions": "not-a-list" + } + + model_response = EmbeddingResponse() + + with pytest.raises(ValueError, match="Expected 'predictions' to be a list"): + VertexBGEConfig.transform_response( + response=response, + model="bge-small-en-v1.5", + model_response=model_response + ) + diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py new file mode 100644 index 0000000000..46f365094c --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_psc_endpoint_support.py @@ -0,0 +1,258 @@ +""" +Unit tests for Vertex AI Private Service Connect (PSC) endpoint support + +Tests that LiteLLM properly constructs URLs when using custom api_base +for PSC endpoints. +""" + +import pytest +import sys +import os + +# Add the litellm package to the path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../..")) + +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + + +class TestVertexAIPSCEndpointSupport: + """Test cases for PSC endpoint URL construction""" + + def test_psc_endpoint_url_construction_basic(self): + """Test basic PSC endpoint URL construction for predict endpoint""" + vertex_base = VertexBase() + psc_api_base = "http://10.96.32.8" + endpoint_id = "1234567890" + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header="test-token", + url="", # This will be replaced + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_psc_endpoint_url_construction_with_streaming(self): + """Test PSC endpoint URL construction with streaming enabled""" + vertex_base = VertexBase() + psc_api_base = "http://10.96.32.8" + endpoint_id = "1234567890" + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="streamGenerateContent", + stream=True, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:streamGenerateContent?alt=sse" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_psc_endpoint_url_construction_v1beta1(self): + """Test PSC endpoint URL construction with v1beta1 API version""" + vertex_base = VertexBase() + psc_api_base = "http://10.96.32.8" + endpoint_id = "1234567890" + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1beta1", + ) + + expected_url = f"{psc_api_base}/v1beta1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_psc_endpoint_url_with_https(self): + """Test PSC endpoint URL construction with HTTPS""" + vertex_base = VertexBase() + psc_api_base = "https://10.96.32.8" + endpoint_id = "1234567890" + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + expected_url = f"{psc_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_psc_endpoint_with_trailing_slash(self): + """Test that trailing slashes in api_base are handled correctly""" + vertex_base = VertexBase() + psc_api_base = "http://10.96.32.8/" + endpoint_id = "1234567890" + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + # rstrip('/') should remove the trailing slash + expected_url = f"{psc_api_base.rstrip('/')}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_standard_proxy_with_googleapis(self): + """Test that standard proxies with googleapis.com in URL use simple format""" + vertex_base = VertexBase() + proxy_api_base = "https://my-proxy.googleapis.com" + endpoint_id = "gemini-pro" # Not numeric + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=proxy_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=False, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + # Should use simple format: api_base:endpoint + expected_url = f"{proxy_api_base}:generateContent" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_custom_proxy_with_numeric_model(self): + """Test that numeric model IDs trigger PSC-style URL construction""" + vertex_base = VertexBase() + proxy_api_base = "https://my-custom-proxy.example.com" + endpoint_id = "9876543210" # Numeric endpoint ID + project_id = "test-project" + location = "us-central1" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=proxy_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header="test-token", + url="", + model=endpoint_id, + vertex_project=project_id, + vertex_location=location, + vertex_api_version="v1", + ) + + # Numeric model should trigger full path construction + expected_url = f"{proxy_api_base}/v1/projects/{project_id}/locations/{location}/endpoints/{endpoint_id}:predict" + assert ( + url == expected_url + ), f"Expected {expected_url}, but got {url}" + + def test_no_api_base_returns_original_url(self): + """Test that when api_base is None, the original URL is returned""" + vertex_base = VertexBase() + original_url = "https://us-central1-aiplatform.googleapis.com/v1/projects/test/locations/us-central1/publishers/google/models/gemini-pro:generateContent" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=None, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="generateContent", + stream=False, + auth_header="test-token", + url=original_url, + model="gemini-pro", + vertex_project="test-project", + vertex_location="us-central1", + vertex_api_version="v1", + ) + + # When api_base is None, original URL should be returned unchanged + assert url == original_url, f"Expected {original_url}, but got {url}" + + def test_auth_header_preserved(self): + """Test that auth_header is properly preserved""" + vertex_base = VertexBase() + psc_api_base = "http://10.96.32.8" + test_auth_header = "Bearer test-token-12345" + + auth_header, url = vertex_base._check_custom_proxy( + api_base=psc_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=False, + auth_header=test_auth_header, + url="", + model="1234567890", + vertex_project="test-project", + vertex_location="us-central1", + vertex_api_version="v1", + ) + + assert ( + auth_header == test_auth_header + ), f"Auth header should be preserved, got {auth_header}" + From 911a0098690158691f4b9884125cb67c73069a51 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Nov 2025 13:28:01 -0800 Subject: [PATCH 17/19] [Docs] LiteLLM Quick start - show how model resolution works (#16602) * docs nderstanding Model Configuration * docs fix --- .../docs/proxy/docker_quick_start.md | 140 +++++++++++++++++- ...odel_prices_and_context_window_backup.json | 3 + 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 7e380e8308..d82a0b01d1 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# E2E Tutorial +# Getting Started Tutorial End-to-End tutorial for LiteLLM Proxy to: - Add an Azure OpenAI model @@ -82,6 +82,8 @@ model_list: ### Model List Specification +You can read more about how model resolution works in the [Model Configuration](#understanding-model-configuration) section. + - **`model_name`** (`str`) - This field should contain the name of the model as received. - **`litellm_params`** (`dict`) [See All LiteLLM Params](https://github.com/BerriAI/litellm/blob/559a6ad826b5daef41565f54f06c739c8c068b28/litellm/types/router.py#L222) - **`model`** (`str`) - Specifies the model name to be sent to `litellm.acompletion` / `litellm.aembedding`, etc. This is the identifier used by LiteLLM to route to the correct model + provider logic on the backend. @@ -89,6 +91,10 @@ model_list: - **`api_base`** (`str`) - The API base for your azure deployment. - **`api_version`** (`str`) - The API Version to use when calling Azure's OpenAI API. Get the latest Inference API version [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation?source=recommendations#latest-preview-api-releases). +--- + + +--- ### Useful Links - [**All Supported LLM API Providers (OpenAI/Bedrock/Vertex/etc.)**](../providers/) @@ -407,6 +413,138 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ - [Set Budgets / Rate Limits per key/user/teams](./users.md) - [Dynamic TPM/RPM Limits for keys](./team_budgets.md#dynamic-tpmrpm-allocation) +## Key Concepts + +This section explains key concepts on LiteLLM AI Gateway. + +### Understanding Model Configuration + +For this config.yaml example: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: azure/my_azure_deployment + api_base: os.environ/AZURE_API_BASE + api_key: "os.environ/AZURE_API_KEY" + api_version: "2025-01-01-preview" # [OPTIONAL] litellm uses the latest azure api_version by default +``` + +**How Model Resolution Works:** + +``` +Client Request LiteLLM Proxy Provider API +────────────── ──────────────── ───────────── + +POST /chat/completions +{ 1. Looks up model_name + "model": "gpt-4o" ──────────▶ in config.yaml + ... +} 2. Finds matching entry: + model_name: gpt-4o + + 3. Extracts litellm_params: + model: azure/my_azure_deployment + api_base: https://... + api_key: sk-... + + 4. Routes to provider ──▶ Azure OpenAI API + POST /deployments/my_azure_deployment/... +``` + +**Breaking Down the `model` Parameter under `litellm_params`:** + +```yaml +model_list: + - model_name: gpt-4o # What the client calls + litellm_params: + model: azure/my_azure_deployment # / + ───── ─────────────────── + │ │ + │ └─────▶ Model name sent to the provider API + │ + └─────────────────▶ Provider that LiteLLM routes to +``` + +**Visual Breakdown:** + +``` +model: azure/my_azure_deployment + └─┬─┘ └─────────┬─────────┘ + │ │ + │ └────▶ The actual model identifier that gets sent to Azure + │ (e.g., your deployment name, or the model name) + │ + └──────────────────▶ Tells LiteLLM which provider to use + (azure, openai, anthropic, bedrock, etc.) +``` + +**Key Concepts:** + +- **`model_name`**: The alias your client uses to call the model. This is what you send in your API requests (e.g., `gpt-4o`). + +- **`model` (in litellm_params)**: Format is `/` + - **Provider** (before `/`): Routes to the correct LLM provider (e.g., `azure`, `openai`, `anthropic`, `bedrock`) + - **Model identifier** (after `/`): The actual model/deployment name sent to that provider's API + +**Advanced Configuration Examples:** + +For custom OpenAI-compatible endpoints (e.g., vLLM, Ollama, custom deployments): + +```yaml +model_list: + - model_name: my-custom-model + litellm_params: + model: openai/nvidia/llama-3.2-nv-embedqa-1b-v2 + api_base: http://my-service.svc.cluster.local:8000/v1 + api_key: "sk-1234" +``` + +**Breaking down complex model paths:** + +``` +model: openai/nvidia/llama-3.2-nv-embedqa-1b-v2 + └─┬──┘ └────────────┬────────────────┘ + │ │ + │ └────▶ Full model string sent to the provider API + │ (in this case: "nvidia/llama-3.2-nv-embedqa-1b-v2") + │ + └──────────────────────▶ Provider (openai = OpenAI-compatible API) +``` + +The key point: Everything after the first `/` is passed as-is to the provider's API. + +**Common Patterns:** + +```yaml +model_list: + # Azure deployment + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-deployment + api_base: https://my-azure.openai.azure.com + + # OpenAI + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + + # Custom OpenAI-compatible endpoint + - model_name: my-llama-model + litellm_params: + model: openai/meta/llama-3-8b + api_base: http://my-vllm-server:8000/v1 + api_key: "optional-key" + + # Bedrock + - model_name: claude-3 + litellm_params: + model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 + aws_region_name: us-east-1 +``` + ## Troubleshooting diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7b8177ed99..fa36e2d608 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9963,6 +9963,7 @@ "supports_function_calling": false, "supports_parallel_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": false, "supports_system_messages": true, "supports_tool_choice": true, @@ -11568,6 +11569,7 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_prompt_caching": true, + "supports_reasoning": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -11670,6 +11672,7 @@ "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, + "supports_reasoning": false, "max_images_per_prompt": 3000, "max_input_tokens": 32768, "max_output_tokens": 32768, From 4be372eb48dbf4cb445d5907c98073acaec351fd Mon Sep 17 00:00:00 2001 From: Nicholas Couture <52148348+orolega@users.noreply.github.com> Date: Fri, 14 Nov 2025 09:30:46 +1100 Subject: [PATCH 18/19] fix: support Anthropic tool_use and tool_result in token counter (#16351) * fix: support Anthropic tool_use and tool_result in token counter * refactor(token_counter): add dynamic field inference for Anthropic content blocks * test: Add additional tests * make format * Fix lint error * Fix mypy narrow type lint errors --- litellm/litellm_core_utils/token_counter.py | 187 ++++++++++-- .../litellm_core_utils/test_token_counter.py | 266 ++++++++++++++++++ 2 files changed, 424 insertions(+), 29 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index fab2c1e76e..a21ebd56f6 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,17 @@ import base64 import io import struct -from typing import Callable, List, Literal, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, + cast, +) import tiktoken @@ -20,6 +30,10 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.default_encoding import encoding as default_encoding from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.types.llms.anthropic import ( + AnthropicMessagesToolResultParam, + AnthropicMessagesToolUseParam, +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionNamedToolChoiceParam, @@ -552,6 +566,131 @@ def _fix_model_name(model: str) -> str: return "gpt-3.5-turbo" +def _count_image_tokens( + image_url: Any, + use_default_image_token_count: bool, +) -> int: + """ + Count tokens for an image_url content block. + + Args: + image_url: The image URL data - can be a string URL or dict with 'url' and 'detail' + use_default_image_token_count: Whether to use default image token counts + + Returns: + int: Number of tokens for the image + + Raises: + ValueError: If image_url is invalid type or detail value is invalid + """ + if isinstance(image_url, dict): + detail = image_url.get("detail", "auto") + if detail not in ["low", "high", "auto"]: + raise ValueError( + f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'." + ) + url = image_url.get("url") + if not url: + raise ValueError("Missing required key 'url' in image_url dict.") + return calculate_img_tokens( + data=url, + mode=detail, # type: ignore + use_default_image_token_count=use_default_image_token_count, + ) + elif isinstance(image_url, str): + if not image_url.strip(): + raise ValueError("Empty image_url string is not valid.") + return calculate_img_tokens( + data=image_url, + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + else: + raise ValueError( + f"Invalid image_url type: {type(image_url).__name__}. " + "Expected str or dict with 'url' field." + ) + + +def _validate_anthropic_content(content: Mapping[str, Any]) -> type: + """ + Validate and determine which Anthropic TypedDict applies. + + Returns the corresponding TypedDict class if recognized, otherwise raises. + """ + content_type = content.get("type") + if not content_type: + raise ValueError("Anthropic content missing required field: 'type'") + + mapping = { + "tool_use": AnthropicMessagesToolUseParam, + "tool_result": AnthropicMessagesToolResultParam, + } + + expected_cls = mapping.get(content_type) + if expected_cls is None: + raise ValueError(f"Unknown Anthropic content type: '{content_type}'") + + missing = [ + k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content + ] + if missing: + raise ValueError( + f"Missing required fields in {content_type} block: {', '.join(missing)}" + ) + + return expected_cls + + +def _count_anthropic_content( + content: Mapping[str, Any], + count_function: TokenCounterFunction, + use_default_image_token_count: bool, + default_token_count: Optional[int], +) -> int: + """ + Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). + + Uses TypedDict definitions from litellm.types.llms.anthropic to determine + what fields to count and how to handle nested structures. + + Dynamically infers which fields to count based on the TypedDict definition, + avoiding hardcoded field names. + """ + typeddict_cls = _validate_anthropic_content(content) + type_hints = getattr(typeddict_cls, "__annotations__", {}) + tokens = 0 + + # Fields to skip (metadata/identifiers that don't contribute to prompt tokens) + skip_fields = {"type", "id", "tool_use_id", "cache_control", "is_error"} + + # Iterate over all fields defined in the TypedDict + for field_name, field_type in type_hints.items(): + if field_name in skip_fields: + continue + + field_value = content.get(field_name) + if field_value is None: + continue + try: + if isinstance(field_value, str): + tokens += count_function(field_value) + elif isinstance(field_value, list): + tokens += _count_content_list( + count_function, + field_value, # type: ignore + use_default_image_token_count, + default_token_count, + ) + elif isinstance(field_value, dict): + tokens += count_function(str(field_value)) + except Exception as e: + if default_token_count is not None: + return default_token_count + raise ValueError(f"Error counting field '{field_name}': {e}") + return tokens + + def _count_content_list( count_function: TokenCounterFunction, content_list: OpenAIMessageContent, @@ -559,7 +698,7 @@ def _count_content_list( default_token_count: Optional[int], ) -> int: """ - Get the number of tokens from a list of content. + Recursively count tokens from a list of content blocks. """ try: num_tokens = 0 @@ -567,42 +706,32 @@ def _count_content_list( if isinstance(c, str): num_tokens += count_function(c) elif c["type"] == "text": - num_tokens += count_function(c["text"]) + num_tokens += count_function(c.get("text", "")) elif c["type"] == "image_url": - if isinstance(c["image_url"], dict): - image_url_dict = c["image_url"] - detail = image_url_dict.get("detail", "auto") - if detail not in ["low", "high", "auto"]: - raise ValueError( - f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'." - ) - url = image_url_dict.get("url") - num_tokens += calculate_img_tokens( - data=url, - mode=detail, # type: ignore - use_default_image_token_count=use_default_image_token_count, - ) - elif isinstance(c["image_url"], str): - image_url_str = c["image_url"] - num_tokens += calculate_img_tokens( - data=image_url_str, - mode="auto", - use_default_image_token_count=use_default_image_token_count, - ) - else: - raise ValueError( - f"Invalid image_url type: {type(c['image_url'])}. Expected str or dict." - ) + image_url = c.get("image_url") + num_tokens += _count_image_tokens( + image_url, use_default_image_token_count + ) + elif c["type"] in ("tool_use", "tool_result"): + num_tokens += _count_anthropic_content( + c, + count_function, + use_default_image_token_count, + default_token_count, + ) else: raise ValueError( - f"Invalid content type: {type(c)}. Expected str or dict." + f"Invalid content item type: {type(c).__name__}. " + f"Expected str or dict with 'type' field. " + f"Value: {c!r}" ) return num_tokens except Exception as e: if default_token_count is not None: return default_token_count raise ValueError( - f"Error getting number of tokens from content list: {e}, default_token_count={default_token_count}" + f"Error getting number of tokens from content list: {e}, " + f"default_token_count={default_token_count}" ) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 5d17ea3dc3..8cd623267b 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -631,3 +631,269 @@ def test_bad_input_token_counter(model, messages): messages=messages, default_token_count=1000, ) + + +def test_token_counter_with_anthropic_tool_use(): + """ + Test that _count_anthropic_content() correctly handles tool_use blocks. + + Validates that: + - 'name' field is counted (string) + - 'input' field is counted (dict serialized to string) + - Metadata fields ('type', 'id') are skipped + """ + messages = [ + { + "role": "user", + "content": "What's the weather in San Francisco?" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll check the weather for you." + }, + { + "type": "tool_use", + "id": "toolu_01234567890", # Should be skipped + "name": "get_weather", # Should be counted + "input": { # Should be counted (serialized) + "location": "San Francisco, CA", + "unit": "fahrenheit" + } + } + ] + } + ] + + tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + assert tokens > 0, f"Expected positive token count, got {tokens}" + # Should count: user message + "I'll check" text + "get_weather" name + input dict + assert tokens > 15, f"Expected reasonable token count for message with tool_use, got {tokens}" + + +def test_token_counter_with_anthropic_tool_result(): + """ + Test that _count_anthropic_content() correctly handles tool_result blocks. + + Validates that: + - 'content' field (when string) is counted + - Metadata fields ('type', 'tool_use_id') are skipped + - Full conversation with tool_use → tool_result flow works + """ + messages = [ + { + "role": "user", + "content": "What's the weather in San Francisco?" + }, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01234567890", + "name": "get_weather", + "input": { + "location": "San Francisco, CA" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01234567890", # Should be skipped + "content": "The weather in San Francisco is 65°F and sunny." # Should be counted + } + ] + } + ] + + tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + assert tokens > 0, f"Expected positive token count, got {tokens}" + assert tokens > 25, f"Expected reasonable token count for conversation with tool_result, got {tokens}" + + +def test_token_counter_with_nested_tool_result(): + """ + Test that _count_anthropic_content() recursively handles nested content lists. + + Validates that: + - tool_result with 'content' as a list (not string) is handled + - Nested content blocks are recursively counted via _count_content_list() + - TypedDict inference correctly identifies list fields + """ + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01234567890", + "content": [ # Nested list - should recursively count + { + "type": "text", + "text": "The weather in San Francisco is 65°F and sunny." + }, + { + "type": "text", + "text": "UV index is moderate." + } + ] + } + ] + } + ] + + tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + assert tokens > 0, f"Expected positive token count, got {tokens}" + # Should count both nested text blocks + assert tokens > 15, f"Expected reasonable token count for nested tool_result, got {tokens}" + + +def test_token_counter_tool_use_and_result_combined(): + """ + Test dynamic field inference with multiple tool_use and tool_result blocks. + + Validates that: + - Multiple tool_use blocks in same message are handled + - Multiple tool_result blocks in same message are handled + - skip_fields correctly filters metadata across all blocks + - Full realistic conversation flow works end-to-end + """ + messages = [ + { + "role": "user", + "content": "What's the weather in San Francisco and New York?" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll check the weather in both cities for you." + }, + { + "type": "tool_use", + "id": "toolu_01A", + "name": "get_weather", + "input": {"location": "San Francisco, CA"} + }, + { + "type": "tool_use", + "id": "toolu_01B", + "name": "get_weather", + "input": {"location": "New York, NY"} + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01A", + "content": "San Francisco: 65°F, sunny" + }, + { + "type": "tool_result", + "tool_use_id": "toolu_01B", + "content": "New York: 45°F, cloudy" + } + ] + }, + { + "role": "assistant", + "content": "The weather in San Francisco is 65°F and sunny, while New York is cooler at 45°F and cloudy." + } + ] + + tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + assert tokens > 0, f"Expected positive token count, got {tokens}" + # Should count all text, tool names, inputs, and results + assert tokens > 60, f"Expected substantial token count for full tool conversation, got {tokens}" + + +def test_token_counter_with_image_url(): + """ + Test that _count_image_tokens() correctly handles image_url content blocks. + + Validates that: + - image_url as dict with 'url' and 'detail' is handled + - image_url as string is handled + - 'detail' field validation works ('low', 'high', 'auto') + - calculate_img_tokens is called with correct parameters + """ + # Test with dict format (detail: low) + messages_dict = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's in this image?" + }, + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "low" # Should use low token count (85 base tokens) + } + } + ] + } + ] + + tokens_dict = token_counter( + model="gpt-3.5-turbo", + messages=messages_dict, + use_default_image_token_count=True # Avoid actual HTTP request + ) + assert tokens_dict > 0, f"Expected positive token count, got {tokens_dict}" + assert tokens_dict > 85, f"Expected at least base image tokens, got {tokens_dict}" + + # Test with string format (defaults to auto/low) + messages_str = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": "https://example.com/image.jpg" # String format + } + ] + } + ] + + tokens_str = token_counter( + model="gpt-3.5-turbo", + messages=messages_str, + use_default_image_token_count=True + ) + assert tokens_str > 0, f"Expected positive token count for string image_url, got {tokens_str}" + + # Test invalid detail value raises error + messages_invalid = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.jpg", + "detail": "invalid" # Should raise ValueError + } + } + ] + } + ] + + try: + token_counter(model="gpt-3.5-turbo", messages=messages_invalid) + assert False, "Expected ValueError for invalid detail value" + except ValueError as e: + assert "Invalid detail value" in str(e), f"Expected detail validation error, got: {e}" + From 124ba463f833889f66289a850511c560a99c5448 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Nov 2025 14:32:09 -0800 Subject: [PATCH 19/19] [Feat] RunwayML - Add support for /audio/speech `eleven_multilingual_v2 ` endpoint (#16604) * init RunwayMLTextToSpeechConfig * add RunwayMLTextToSpeechConfig * add RunwayMLTextToSpeechConfig * test_runwayml_tts_async * runway ml speech * fix voices * fix test * docs runway lm * add runwayml here * fix RunwayMLTextToSpeechConfig * test_openai_voice_mapping_to_runwayml --- .../docs/providers/runwayml/text-to-speech.md | 244 ++++++++ .../llms/runwayml/text_to_speech/__init__.py | 5 + .../runwayml/text_to_speech/transformation.py | 591 ++++++++++++++++++ .../llms/runwayml/videos/transformation.py | 9 +- litellm/main.py | 33 + litellm/utils.py | 6 + provider_endpoints_support.json | 2 +- tests/audio_tests/runwayml_speech.mp3 | Bin 0 -> 67503 bytes tests/audio_tests/test_audio_speech.py | 54 ++ .../test_text_to_speech_transformation.py | 67 ++ 10 files changed, 1003 insertions(+), 8 deletions(-) create mode 100644 docs/my-website/docs/providers/runwayml/text-to-speech.md create mode 100644 litellm/llms/runwayml/text_to_speech/__init__.py create mode 100644 litellm/llms/runwayml/text_to_speech/transformation.py create mode 100644 tests/audio_tests/runwayml_speech.mp3 create mode 100644 tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py diff --git a/docs/my-website/docs/providers/runwayml/text-to-speech.md b/docs/my-website/docs/providers/runwayml/text-to-speech.md new file mode 100644 index 0000000000..020269863c --- /dev/null +++ b/docs/my-website/docs/providers/runwayml/text-to-speech.md @@ -0,0 +1,244 @@ +# RunwayML - Text-to-Speech + +## Overview + +| Property | Details | +|-------|-------| +| Description | RunwayML provides high-quality AI-powered text-to-speech with natural-sounding voices | +| Provider Route on LiteLLM | `runwayml/` | +| Supported Operations | [`/audio/speech`](#quick-start) | +| Link to Provider Doc | [RunwayML API ↗](https://docs.dev.runwayml.com/) | + +LiteLLM supports RunwayML's text-to-speech API with automatic task polling, allowing you to generate natural-sounding audio from text. + +## Quick Start + +```python showLineNumbers title="Basic Text-to-Speech" +from litellm import speech +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Step right up, ladies and gentlemen! Have you ever wished for a toaster that's not just a toaster but a marvel of modern ingenuity?", + voice="alloy" +) + +# Save the audio +with open("output.mp3", "wb") as f: + f.write(response.content) +``` + +## Authentication + +Set your RunwayML API key: + +```python showLineNumbers title="Set API Key" +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" +``` + +## Supported Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use (e.g., `runwayml/eleven_multilingual_v2`) | +| `input` | string | Yes | Text to convert to speech | +| `voice` | string or dict | Yes | Voice to use (OpenAI name, RunwayML preset, or voice config) | + +## Voice Options + +### Using OpenAI Voice Names + +OpenAI voice names are automatically mapped to appropriate RunwayML voices: + +```python showLineNumbers title="OpenAI Voice Names" +from litellm import speech + +# These OpenAI voice names work automatically +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="alloy" # Maya - neutral, balanced female voice +) +``` + +**Voice Mappings:** +- `alloy` → Maya (neutral, balanced female voice) +- `echo` → James (male voice) +- `fable` → Bernard (warm, storytelling voice) +- `onyx` → Vincent (deep male voice) +- `nova` → Serene (warm, expressive female voice) +- `shimmer` → Ella (clear, friendly female voice) + +### Using RunwayML Preset Voices + +You can directly specify any RunwayML preset voice by passing the preset name as a string: + +```python showLineNumbers title="RunwayML Preset Names" +from litellm import speech + +# Pass the RunwayML voice name as a string +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="Maya" # LiteLLM automatically formats this for RunwayML +) + +# Try different RunwayML voices +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Step right up, ladies and gentlemen!", + voice="Bernard" # Great for storytelling +) +``` + +**Available RunwayML Voices:** + +Maya, Arjun, Serene, Bernard, Billy, Mark, Clint, Mabel, Chad, Leslie, Eleanor, Elias, Elliot, Grungle, Brodie, Sandra, Kirk, Kylie, Lara, Lisa, Malachi, Marlene, Martin, Miriam, Monster, Paula, Pip, Rusty, Ragnar, Xylar, Maggie, Jack, Katie, Noah, James, Rina, Ella, Mariah, Frank, Claudia, Niki, Vincent, Kendrick, Myrna, Tom, Wanda, Benjamin, Kiana, Rachel + +:::tip +Simply pass the voice name as a string - LiteLLM automatically handles the internal RunwayML API format conversion. +::: + +## Async Usage + +```python showLineNumbers title="Async Text-to-Speech" +from litellm import aspeech +import os +import asyncio + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +async def generate_speech(): + response = await aspeech( + model="runwayml/eleven_multilingual_v2", + input="This is an asynchronous text-to-speech request.", + voice="nova" + ) + + with open("output.mp3", "wb") as f: + f.write(response.content) + + print("Audio generated successfully!") + +asyncio.run(generate_speech()) +``` + +## LiteLLM Proxy Usage + +Add RunwayML to your proxy configuration: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: runway-tts + litellm_params: + model: runwayml/eleven_multilingual_v2 + api_key: os.environ/RUNWAYML_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +Generate speech through the proxy: + +```bash showLineNumbers title="Proxy Request" +curl --location 'http://localhost:4000/v1/audio/speech' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/eleven_multilingual_v2", + "input": "Hello from the LiteLLM proxy!", + "voice": "alloy" +}' +``` + +With RunwayML-specific voice: + +```bash showLineNumbers title="Proxy Request with RunwayML Voice" +curl --location 'http://localhost:4000/v1/audio/speech' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/eleven_multilingual_v2", + "input": "Hello with a custom RunwayML voice!", + "voice": "Bernard" +}' +``` + +## Supported Models + +| Model | Description | +|-------|-------------| +| `runwayml/eleven_multilingual_v2` | High-quality multilingual text-to-speech | + +## Cost Tracking + +LiteLLM automatically tracks RunwayML text-to-speech costs: + +```python showLineNumbers title="Cost Tracking" +from litellm import speech, completion_cost + +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="alloy" +) + +cost = completion_cost(completion_response=response) +print(f"Text-to-speech cost: ${cost}") +``` + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Text-to-Speech | ✅ | +| Cost Tracking | ✅ | +| Logging | ✅ | +| Fallbacks | ✅ | +| Load Balancing | ✅ | +| 50+ Voice Presets | ✅ | + +## How It Works + +RunwayML uses an asynchronous task-based API pattern. LiteLLM handles the polling and response transformation automatically. + +### Complete Flow Diagram + +```mermaid +sequenceDiagram + participant Client + box rgb(200, 220, 255) LiteLLM AI Gateway + participant LiteLLM + end + participant RunwayML as RunwayML API + participant Storage as Audio Storage + + Client->>LiteLLM: POST /audio/speech (OpenAI format) + Note over LiteLLM: Transform to RunwayML format
Map voice to preset ID + + LiteLLM->>RunwayML: POST v1/text_to_speech + RunwayML-->>LiteLLM: 200 OK + task ID + + Note over LiteLLM: Automatic Polling + loop Every 2 seconds + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: RUNNING + end + + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: SUCCEEDED + audio URL + + LiteLLM->>Storage: GET audio URL + Storage-->>LiteLLM: Audio data (MP3) + + Note over LiteLLM: Return audio content + LiteLLM-->>Client: Audio Response (binary) +``` + diff --git a/litellm/llms/runwayml/text_to_speech/__init__.py b/litellm/llms/runwayml/text_to_speech/__init__.py new file mode 100644 index 0000000000..491e8449e0 --- /dev/null +++ b/litellm/llms/runwayml/text_to_speech/__init__.py @@ -0,0 +1,5 @@ +"""RunwayML Text-to-Speech implementation.""" +from .transformation import RunwayMLTextToSpeechConfig + +__all__ = ["RunwayMLTextToSpeechConfig"] + diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py new file mode 100644 index 0000000000..ac926beb22 --- /dev/null +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -0,0 +1,591 @@ +""" +RunwayML Text-to-Speech transformation + +Maps OpenAI TTS spec to RunwayML Text-to-Speech API +""" +import asyncio +import time +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union + +import httpx + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import ( + RUNWAYML_DEFAULT_API_VERSION, + RUNWAYML_POLLING_TIMEOUT, +) +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + HttpxBinaryResponseContent = Any + + +class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): + """ + Configuration for RunwayML Text-to-Speech + + Reference: https://api.dev.runwayml.com/v1/text_to_speech + """ + + DEFAULT_BASE_URL: str = "https://api.dev.runwayml.com" + TTS_ENDPOINT_PATH: str = "v1/text_to_speech" + DEFAULT_MODEL: str = "eleven_multilingual_v2" + DEFAULT_VOICE_TYPE: str = "runway-preset" + DEFAULT_VOICE_PRESET_ID: str = "Bernard" + + # Voice mappings from OpenAI voices to RunwayML preset IDs + # OpenAI voices mapped to similar-sounding RunwayML voices + VOICE_MAPPINGS = { + "alloy": "Maya", # Neutral, balanced female voice + "echo": "James", # Male voice + "fable": "Bernard", # Warm, storytelling voice + "onyx": "Vincent", # Deep male voice + "nova": "Serene", # Warm, expressive female voice + "shimmer": "Ella", # Clear, friendly female voice + } + + def dispatch_text_to_speech( + self, + model: str, + input: str, + voice: Optional[Union[str, Dict]], + optional_params: Dict, + litellm_params_dict: Dict, + logging_obj: "LiteLLMLoggingObj", + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]], + base_llm_http_handler: Any, + aspeech: bool, + api_base: Optional[str], + api_key: Optional[str], + **kwargs: Any, + ) -> Union[ + "HttpxBinaryResponseContent", + Coroutine[Any, Any, "HttpxBinaryResponseContent"], + ]: + """ + Dispatch method to handle RunwayML TTS requests + + This method encapsulates RunwayML-specific credential resolution and parameter handling + + Args: + base_llm_http_handler: The BaseLLMHTTPHandler instance from main.py + """ + # Resolve api_base from multiple sources + api_base = ( + api_base + or litellm_params_dict.get("api_base") + or litellm.api_base + or get_secret_str("RUNWAYML_API_BASE") + or self.DEFAULT_BASE_URL + ) + + # Resolve api_key from multiple sources + api_key = ( + api_key + or litellm_params_dict.get("api_key") + or litellm.api_key + or get_secret_str("RUNWAYML_API_SECRET") + or get_secret_str("RUNWAYML_API_KEY") + ) + + # Convert voice to appropriate format + voice_param: Optional[Union[str, Dict]] = voice + if isinstance(voice, str): + # Keep as string, will be processed in map_openai_params + voice_param = voice + elif isinstance(voice, dict): + # Already in dict format, pass through + voice_param = voice + + litellm_params_dict.update({ + "api_key": api_key, + "api_base": api_base, + }) + + # Call the text_to_speech_handler + response = base_llm_http_handler.text_to_speech_handler( + model=model, + input=input, + voice=voice_param, + text_to_speech_provider_config=self, + text_to_speech_optional_params=optional_params, + custom_llm_provider="runwayml", + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=None, + _is_async=aspeech, + ) + + return response + + def get_supported_openai_params(self, model: str) -> list: + """ + RunwayML TTS supports these OpenAI parameters + """ + return ["voice"] + + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Dict = {}, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI parameters to RunwayML TTS parameters + + Returns: + Tuple of (mapped_voice_string, mapped_params) + + Note: Since RunwayML requires voice as a dict, we store it in + mapped_params["runwayml_voice"] and return None for the voice string. + """ + mapped_params = {} + + # Map voice parameter to RunwayML format dict + voice_dict: Optional[Dict] = None + if isinstance(voice, str): + # Check if it's an OpenAI voice name that needs mapping + if voice in self.VOICE_MAPPINGS: + preset_id = self.VOICE_MAPPINGS[voice] + voice_dict = { + "type": self.DEFAULT_VOICE_TYPE, + "presetId": preset_id, + } + else: + # Assume it's a RunwayML preset ID + voice_dict = { + "type": self.DEFAULT_VOICE_TYPE, + "presetId": voice, + } + elif isinstance(voice, dict): + # Already in RunwayML format, use as-is + voice_dict = voice + + # Store the voice dict in optional_params for later use + if voice_dict is not None: + mapped_params["runwayml_voice"] = voice_dict + + # No other OpenAI params are currently supported by RunwayML TTS + # (response_format, speed, etc. are not supported) + + # Return None for voice string since RunwayML uses dict format + return None, mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate RunwayML environment and set up authentication headers + """ + validated_headers = headers.copy() + + final_api_key = ( + api_key + or get_secret_str("RUNWAYML_API_SECRET") + or get_secret_str("RUNWAYML_API_KEY") + ) + + if not final_api_key: + raise ValueError("RUNWAYML_API_SECRET or RUNWAYML_API_KEY is not set") + + validated_headers["Authorization"] = f"Bearer {final_api_key}" + validated_headers["X-Runway-Version"] = RUNWAYML_DEFAULT_API_VERSION + validated_headers["Content-Type"] = "application/json" + + return validated_headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for RunwayML TTS request + """ + complete_url = ( + api_base + or get_secret_str("RUNWAYML_API_BASE") + or self.DEFAULT_BASE_URL + ) + + complete_url = complete_url.rstrip("/") + return f"{complete_url}/{self.TTS_ENDPOINT_PATH}" + + @staticmethod + def _check_timeout(start_time: float, timeout_secs: float) -> None: + """ + Check if operation has timed out. + + Args: + start_time: Start time of the operation + timeout_secs: Timeout duration in seconds + + Raises: + TimeoutError: If operation has exceeded timeout + """ + if time.time() - start_time > timeout_secs: + raise TimeoutError( + f"RunwayML TTS task polling timed out after {timeout_secs} seconds" + ) + + @staticmethod + def _check_task_status(response_data: Dict[str, Any]) -> str: + """ + Check RunwayML task status from response. + + RunwayML statuses: PENDING, RUNNING, SUCCEEDED, FAILED, CANCELLED, THROTTLED + + Args: + response_data: JSON response from RunwayML task endpoint + + Returns: + Normalized status string: "running", "succeeded", or raises on failure + + Raises: + ValueError: If task failed or status is unknown + """ + status = response_data.get("status", "").upper() + + verbose_logger.debug(f"RunwayML TTS task status: {status}") + + if status == "SUCCEEDED": + return "succeeded" + elif status == "FAILED": + failure_reason = response_data.get("failure", "Unknown error") + failure_code = response_data.get("failureCode", "unknown") + raise ValueError( + f"RunwayML TTS failed: {failure_reason} (code: {failure_code})" + ) + elif status == "CANCELLED": + raise ValueError("RunwayML TTS was cancelled") + elif status in ["PENDING", "RUNNING", "THROTTLED"]: + return "running" + else: + raise ValueError(f"Unknown RunwayML task status: {status}") + + def _poll_task_sync( + self, + task_id: str, + api_base: str, + headers: Dict[str, str], + timeout_secs: float = 600, + ) -> httpx.Response: + """ + Poll RunwayML task until completion (sync). + + RunwayML POST returns immediately with a task that has status PENDING/RUNNING. + We need to poll GET /v1/tasks/{task_id} until status is SUCCEEDED or FAILED. + + Args: + task_id: The task ID to poll + api_base: Base URL for RunwayML API + headers: Request headers (including auth) + timeout_secs: Total timeout in seconds (default: 600s = 10 minutes) + + Returns: + Final response with completed task + """ + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client() + start_time = time.time() + + # Build task status URL + api_base = api_base.rstrip("/") + task_url = f"{api_base}/v1/tasks/{task_id}" + + verbose_logger.debug(f"Polling RunwayML TTS task: {task_url}") + + while True: + self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) + + # Poll the task status + response = client.get(url=task_url, headers=headers) + response.raise_for_status() + + response_data = response.json() + + # Check task status + status = self._check_task_status(response_data=response_data) + + if status == "succeeded": + return response + elif status == "running": + # Wait before polling again (RunwayML recommends 1-2 second intervals) + time.sleep(2) + + async def _poll_task_async( + self, + task_id: str, + api_base: str, + headers: Dict[str, str], + timeout_secs: float = 600, + ) -> httpx.Response: + """ + Poll RunwayML task until completion (async). + + Args: + task_id: The task ID to poll + api_base: Base URL for RunwayML API + headers: Request headers (including auth) + timeout_secs: Total timeout in seconds (default: 600s = 10 minutes) + + Returns: + Final response with completed task + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client = get_async_httpx_client(llm_provider=litellm.LlmProviders.RUNWAYML) + start_time = time.time() + + # Build task status URL + api_base = api_base.rstrip("/") + task_url = f"{api_base}/v1/tasks/{task_id}" + + verbose_logger.debug(f"Polling RunwayML TTS task (async): {task_url}") + + while True: + self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) + + # Poll the task status + response = await client.get(url=task_url, headers=headers) + response.raise_for_status() + + response_data = response.json() + + # Check task status + status = self._check_task_status(response_data=response_data) + + if status == "succeeded": + return response + elif status == "running": + # Wait before polling again (RunwayML recommends 1-2 second intervals) + await asyncio.sleep(2) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[Union[str, Dict]], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Transform OpenAI TTS request to RunwayML TTS format + + RunwayML expects: + - model: The model to use (e.g., 'eleven_multilingual_v2') + - promptText: The text to convert to speech + - voice: Voice configuration object + { + "type": "runway-preset", + "presetId": "Bernard" + } + + Returns: + TextToSpeechRequestData: Contains JSON body and headers + """ + # Get voice from optional_params (mapped in map_openai_params) + runwayml_voice = optional_params.get("runwayml_voice") + if runwayml_voice is None: + # Use default voice if not provided + runwayml_voice = { + "type": self.DEFAULT_VOICE_TYPE, + "presetId": self.DEFAULT_VOICE_PRESET_ID, + } + + # Build request body + request_body = { + "model": model or self.DEFAULT_MODEL, + "promptText": input, + "voice": runwayml_voice, + } + + # Add any other optional parameters (except runwayml_voice which we already used) + for k, v in optional_params.items(): + if k not in request_body and k != "runwayml_voice": + request_body[k] = v + + return { + "dict_body": request_body, + "headers": headers, + } + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + """ + Transform RunwayML TTS response to standard format + + RunwayML returns a task immediately with status PENDING/RUNNING. + We need to poll the task until it completes, then download the audio. + + Initial response: + { + "id": "task_123...", + "status": "PENDING" | "RUNNING", + "createdAt": "2025-11-13T..." + } + + After polling: + { + "id": "task_123...", + "status": "SUCCEEDED", + "output": ["https://storage.googleapis.com/.../audio.mp3"], + "completedAt": "2025-11-13T..." + } + """ + from litellm.types.llms.openai import HttpxBinaryResponseContent + + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing RunwayML TTS response: {e}", + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + + verbose_logger.debug("RunwayML TTS starting polling...") + + # Get task ID + task_id = response_data.get("id") + if not task_id: + raise ValueError("RunwayML TTS response missing task ID") + + # Get headers for polling (need auth) + poll_headers = { + "Authorization": raw_response.request.headers.get("Authorization", ""), + "X-Runway-Version": raw_response.request.headers.get( + "X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION + ), + } + + # Poll until task completes + polled_response = self._poll_task_sync( + task_id=task_id, + api_base=self.DEFAULT_BASE_URL, + headers=poll_headers, + timeout_secs=RUNWAYML_POLLING_TIMEOUT, + ) + + # Get the completed task data + task_data = polled_response.json() + + verbose_logger.debug("RunwayML TTS polling complete, downloading audio") + + # Get audio URL from output + output = task_data.get("output", []) + if not output or not isinstance(output, list) or len(output) == 0: + raise ValueError("RunwayML TTS response missing audio URL in output") + + audio_url = output[0] + if not isinstance(audio_url, str): + raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}") + + # Download the audio file + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client() + audio_response = client.get(url=audio_url) + audio_response.raise_for_status() + + verbose_logger.debug("RunwayML TTS audio downloaded successfully") + + # Return the audio data wrapped in HttpxBinaryResponseContent + return HttpxBinaryResponseContent(audio_response) + + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + """ + Async transform RunwayML TTS response to standard format + + Same as sync version but uses async polling and download + """ + from litellm.types.llms.openai import HttpxBinaryResponseContent + + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing RunwayML TTS response: {e}", + status_code=raw_response.status_code, + headers=dict(raw_response.headers), + ) + + verbose_logger.debug("RunwayML TTS starting polling (async)...") + + # Get task ID + task_id = response_data.get("id") + if not task_id: + raise ValueError("RunwayML TTS response missing task ID") + + # Get headers for polling (need auth) + poll_headers = { + "Authorization": raw_response.request.headers.get("Authorization", ""), + "X-Runway-Version": raw_response.request.headers.get( + "X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION + ), + } + + # Poll until task completes (async) + polled_response = await self._poll_task_async( + task_id=task_id, + api_base=self.DEFAULT_BASE_URL, + headers=poll_headers, + timeout_secs=RUNWAYML_POLLING_TIMEOUT, + ) + + # Get the completed task data + task_data = polled_response.json() + + verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio") + + # Get audio URL from output + output = task_data.get("output", []) + if not output or not isinstance(output, list) or len(output) == 0: + raise ValueError("RunwayML TTS response missing audio URL in output") + + audio_url = output[0] + if not isinstance(audio_url, str): + raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}") + + # Download the audio file (async) + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client = get_async_httpx_client(llm_provider=litellm.LlmProviders.RUNWAYML) + audio_response = await client.get(url=audio_url) + audio_response.raise_for_status() + + verbose_logger.debug("RunwayML TTS audio downloaded successfully (async)") + + # Return the audio data wrapped in HttpxBinaryResponseContent + return HttpxBinaryResponseContent(audio_response) + diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index c45a1cd60d..651acff6fc 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -6,6 +6,8 @@ from httpx._types import RequestFiles import litellm from litellm.constants import RUNWAYML_DEFAULT_API_VERSION +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -23,16 +25,9 @@ from litellm.types.videos.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException - from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig - LiteLLMLoggingObj = _LiteLLMLoggingObj - BaseVideoConfig = _BaseVideoConfig - BaseLLMException = _BaseLLMException else: LiteLLMLoggingObj = Any - BaseVideoConfig = Any - BaseLLMException = Any class RunwayMLVideoConfig(BaseVideoConfig): diff --git a/litellm/main.py b/litellm/main.py index 2ad444a9a2..23ffa90e2d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6006,6 +6006,39 @@ def speech( # noqa: PLR0915 logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, ) + elif custom_llm_provider == "runwayml": + from litellm.llms.runwayml.text_to_speech.transformation import ( + RunwayMLTextToSpeechConfig, + ) + + # RunwayML Text-to-Speech + if text_to_speech_provider_config is None: + raise litellm.BadRequestError( + message="RunwayML Text-to-Speech configuration not found", + model=model, + llm_provider=custom_llm_provider, + ) + + # Cast to specific RunwayML config type to access dispatch method + runwayml_config = cast( + RunwayMLTextToSpeechConfig, text_to_speech_provider_config + ) + + response = runwayml_config.dispatch_text_to_speech( # type: ignore + model=model, + input=input, + voice=voice, + optional_params=optional_params, + litellm_params_dict=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + base_llm_http_handler=base_llm_http_handler, + aspeech=aspeech or False, + api_base=api_base, + api_key=api_key, + **kwargs, + ) if response is None: raise Exception( diff --git a/litellm/utils.py b/litellm/utils.py index 7f87a800b0..ae3de67374 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7811,6 +7811,12 @@ class ProviderConfigManager: ) return AzureAVATextToSpeechConfig() + elif litellm.LlmProviders.RUNWAYML == provider: + from litellm.llms.runwayml.text_to_speech.transformation import ( + RunwayMLTextToSpeechConfig, + ) + + return RunwayMLTextToSpeechConfig() return None @staticmethod diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 6c3be897f4..4d9218d609 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1368,7 +1368,7 @@ "embeddings": false, "image_generations": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "moderations": false, "batches": false, "rerank": false, diff --git a/tests/audio_tests/runwayml_speech.mp3 b/tests/audio_tests/runwayml_speech.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..22aee2084ba131cf077a4fdee3706272f52f861e GIT binary patch literal 67503 zcmd>ljSJbUeR)}se`pF9Bgf0v*Uo04SIr0YFR~JVG!j1r;4V z6AK#$7mt9Dh?uy<3mG{DWi>4w14Cmo8wZzHUOxVTL2nV^Q8DpJsp*+nIeG6s6ciSh zlvme&Y;11r`23}}e`tJaW_E6Ib$xSZ@B87&>Ba9qxAzZ^$iM%kr6#Q|FTl;m%ZuLe zeSAH|1VeMeZk)Yf&oxU{_Ys!2(Y$<08r#`FyOBykxz7V zA;>Mdv$M0Fe{h_go!`~5FebWi#?<_NbsMTwQTM#862#KHpl278v$2tFXsT!W-hvL%hYk;bpihg8rk5o6?#c6_A( z4MyoBgY_QyRfv3$9pzG!x|5wS2Mwgw;`J(pif|-Q>(&@FQ!(D zW{fu=kHX%i!}rEj6JW)brIOX9LEdW(oWOA=ZmG+&9eVwFtnD*-QE+UyBZ>hq0l<(f zvAO^V8v)T~9EtEoLzx50dE@ocClB=0#lr&+w0oZ_82eXZML? zV-85yCyS5JV2m{y@9G#jcHZ5IjK(4b`(SneWhn67KjhUF3A+X{nUJP9x8qHzFkUelo%;xhD;oH*Rr3; znK(V9iO6gui5NoP#nc7k;7GLLiKZUp5REH0@|&2tFVq${2B|b(&-$RF3`DBm&~?hL zu|Tb+-kE3EWMwmQj-S}^#!|j09BJ0}@X-}EqZMGH)%>9wLtq2KDYeDMz>^qGQauA?2Hr8O+q!9{7 z7!1avI!(kY<7!rlO*Z*w>MC+Nl<%3QBk0pi>*>0B*hfeQWk)6WSPV6>dGw&kIR7D5 z@=MFQeS$AF`pkkesqxK9`FiI=$RO{MbjddjWtsWr^!^*O;L>PN)9Uf3Dk4vXCgVgO2Or39~P-IU?A70p^F=r zZ}QHn`uqXpoh2~CV&1$JF7=)eiehb9A%yVS%~m;0OEmU1<@#83Q5gFpQztL}x9G6R zCy75IwOw!BCs*_z-zd2oaUt*F1FLXQM}+PPw8`0l|4=f$sC44?qNzD}=Q+A|Pra(n z>eaQ*Y?6By>aYV%=t;Tx%$fp4-(WoB?j7fp{lU|ΨG%-fafY!m>&%q3nlIsp)a zk3PcyW1kX&I>E|9$40=Ss7;TCOUZBDFCv2x8$(hio=gO@`~f!rjlcSG7r9We${K#L zZ=>4;{eHxU^*BC(+t3ucJ=(wdSo3~E+HNC32*fY!SRt;UBeIw0cq2L zrVXa5)ZH@A!(GF6t3DaDFtp2Y2%?9h->J}TzWcc>5bcLbD!qJ3R$oY1Gd084EjS)*izx1v6RY$}x*6qy)@1JFDKUowv zuTM#Z`J;&c9wYW}18Q>ge(683FW?mUA6_1JC9oHAJ9V_u->&+4EUt6*zbEpNsdSW` ziP|p(9~c`G&~*dIzE`N(4D|ycdltAw!%Og-(=*vz<%{3t1~cX?I?ztFM2 zxUe{k+~z^{%>@=M(lK$8QsJ}>T3Z$wU`jj*&q3_o5_Pknjf%lB22?WaFdjqd{QQFgQqN7@kCK3Hgub}uYaFJC?1qDXHsWXUY2BD=U(G$ zB!^CZTXlu&>RZl_CkjXqjE^wpjY)$$k_h~9X9#dkC-_!&?zr_h0g3NJY4*WZ<9_#~ zC3`KFFURI~Xb_Sc)EEQV3)xz^9JnfT_iQy;LTbKAh+7mjkRbln@9@Wq)c4KdmklIx zJYDZrIt5`wuoKCbFZ}F^Y{gT$v7|WETE>yk5!K@`4Yi>qBGp(Jq;SLu1&{p>Ix&+= z5BANV)goO6YlCy9JVG;>k@p@0-{D~W1iNE9oI@R+>c@AGE?z(b0Pl;nZKq{y!ksM! zg`G7S0End(?<^?m`IdK5QAstRfMU8rEj6{Xui!icp5+r}^yQjFiJl(oP|rOQn9DIb?gX$L+jZu&<$}QA4A&5f8KAO!@@;-c`7P$E-k*3 zzWoIWMSzu|zH0O3nZb7!=YJhFdTeYqrw^_~t=UA7NXUyx0ch`$ZL>~>3M?gk@Y+9t zv9{>1KYz^29;(Xi4RZU((tS9Ic9_X=&C5fT`orLi*jp1=x?G!%y+~k?jJKM_S?yN{ zn8JJnU{*whPdBs@Dvf=GYCXoquh5=yvlfWd$_Bz=D%>%u4Pi?j#;6)}D-1+;MDXdF zO8OHy6StusoV4%R2Vw^nQd1X~t_P=(XXlM|gt)21nEaO2A;{-7UqA83pm=g)8G6#o zwPOw9q5r7;&RC8dLroK>=*v)cM-Aks-~~Sg0j6&`RS}7v3nVmGb-y z7d^J4pNa{&72;7nWsKtHQPyn@k7hPm=H$>IzG7nLfC3LS%BN z@wFdxce@96M`k>6^W3Ddb7B3&v5U5M=163?Wu}m!3?ehW^!+2XO?`js-!b&o&dc(K zoY)vmW>B1E$ql>{Hi13ZsHf!Mg<;BUeNO-eQi2Jc`AcPB69p{lOQY#+m$zy)k=X$S zKVNoLaFNU` z(W2ARY;}*?8MH5Q`gXReue5x7?j=DZ30d3yZalF9N<9Y)%Wg05wk-CWTVs=Z_`q)< zxN%X7DOXrwo=&8~A;?RcF>qX#0-mV8IhLef19C%8zgy5YsjO)Hg(tbh!B9}gu<-n*vc=FNG%Qu)%eK3cQM$5*rebuF@t zy4hErXLs%+yPAn?yr7{9?x13S=S1?0&Q)t9@{j6xv78GjnQ$ZyDRUGPWQnK!Fkm-g zQPs1Y@BTPNX2;#Gp}G9|`R~txY}5-ciR7S3j}cB zk+YI=&KdB*VC4h!(BG7v9gRbYkDW?NXe743fBWig_zqpR{_y$Q>fab`83Y)MB~Kl- z{jKB_jxKWCc&s9yf`tK?W!2yO)t}*hw@&p)Vfdx%HL`Bg+{sw|kNw}}VDC8=G#l?$ ziSeaxh~{J8W$af1dZ+%IWdVam+vCU3!cVq}m(18x{rzd@MAH-7GL$=mr;mj6b4qEe zF|=>f@;B3CZEXJm_XsEMwNgKkB&_mms(frKdWXg!lR}=UDjmv;veg<$k41rs1)@WL zc@jU6Kwx>2ndO5Duo(nwQ)>Krk?QW})p^Ca_Il%--bsMiZ!^&JR&&)%FH>P944hT{ z#=VT__ufs79JXkqGKe`}sV6y^NalIS-4qq#M}dHCe6uGi1CZz$HcF6dQ|({b2|4F{ z>6y>*S~&SA;8{=qaI>u}>FWMxIq9n6BE1ZtgO&`3Yt=9&39|rtY$MCWu^65LW29P* zNkzJ0@k0PaDLMj!f}*+Wb5fXWAX7ts{=H2-mYjUbggyN3XC%A+thBIb+ z_VWrkn0_`aN%}wU4ItnDpndEVbr{OYA=zB4XFvbC7@x3zy)X`|YN$HZPc%hoh8#l= zC(Ag-|`fItEj<39(6Q zo~E=g3lWNTt5R9rbvw2l-9DmY6(?qaf5{ed>WWw;AOg!yh=3w8&T!jmcf8T*xU&Ls zp;qzoPFxe3g5k^DBLqVs2K1t!%@*P^!c}GIF%S#|uoSDwjrvGj0%~i6FCVI`w-&Yk zie$v#2&s+?+=gPL5fPap9Ti{2s&;(Gyzbtrx%qo?bXKVC%^mHbo1GJWBNKP3`^-NR z{;h|)g>hCLM%{EZoOBbAreQ3&Of)B`04t?`X062rT9i_{w6Jn_v$va(U&KsJf^KSu z%7LJ&N~eU;sCcRp4(0+qIJ7JB?& z8)nC`ICb>L@DBw8Yq6v|-mz8VGX&;XDeT~W%LF*qv})4CFGIxI);*kKrR8ewnAI5^ zIz||jd5l;Ou~5&A_DersCdw!Bi{O_|Qp?!h?Q6);RDJpnE94XN&VCaeekP4u0tnF_ zO~z!oJL*l_p-NFJ&;*9P;A{&SZ_f|y=rq+6VdUf#N1IX*t*WvFj&|Xj1)+ow%wA!u zcr9y5=J1mdCkS^TqpEG(r`h3P55Im1r0eu$Vtu%8x1b@`*u z?_m--QpfIxoR_J|7%Zj%xvzC97b}VZ_9Jo_^lkuaUNsME>}XN>;pk}T#t$}B;`3bB zlgsH#q@nVK8qz?Aa{xJAyk%u%J|nu%&mE_dyiaf6x@8-mp4V|X2o`CtC`Og%(n^nY z9zC7X6(7}wXfI66tGY^lo0r7}J93XTMy0E{frs4&-sq`RD>OGCpBkR{>p=_vQj;L5 zR=oy~dPrXrT+j*}pUgx@FB-{-CYUY??D4hiE$9$;qnJ>9u8}2P9QCPMGH?HCxi)fS z+%|!FG_c>82C24KVj#&iAQMiUUi?&s4NV3UB0`_m%_q za3Ws|v*R4l_HA`lR~R`>P*PaBQGWv|R7H;OyUyrU6K$$TL)Ytd(wC1nC(dvF;`ple zh~bV^lUYHE2?qHa%iPE8nKR-6MVgzcfF?t^XL2;W6xM|vtFqkHqa&$WYXkw`gsli? zsaK*L_v=2r{^s$<{&E;_mm5C5WcZusrB&)7#QIHeC+r+?NSGgs5UmgF{~ISGZv$pF z%|GUqr&$j$(Xjqd(jlZ^Z{~gb+^7h1_^geDpTndkWt20c4K2oOw?LY;ZFU>o(nk0 zGfw@)4sFMful#Dbu;t==v_pNvu{KG9qVhsXh?1FMV@OvDFSE)=BvQGhiIh<&;LAo- zv>#*tfcO+SuMn;Xjj84Ywm`>chmron-!&@yr)wkZ03sAsrZXr>s4g%jaiv+Pov4=) zZ7rn_1BSqoY4;#77sFN5#(MqCd1<}+(PSlA-4KTNE_6+OkNGhTce)BN*zqEf$S&=erpDX?+LTWg}v zBD!+qjUBH>Qmp8&>L(@8k2W$692P>lr`uusT3CKd_nj(bc@u1om&zKJKV(Mm-DBtb zF&u!hYyk%tYommwA=C>zajeGZT>Q@{h$%dt6Jy%~SYb-H6)rBmZ)b6DL zDOKA5VgOZ^T`e*|i~O&UU6}?96sla_92e0{`VoQVB|ouc*U@C63*d1zU4rX=(u2{J z`Gic5p$va-(-ODo7(>(b<+p&<{5C1b*d#h1^Au-+ zFbj3KNmu^*QyTKuzA7~4S8Cv#spGg^lsMC-{y|o8jy~WQ0|jh3&sbN81E*9X`PWFc zGdr|Bd2zNw&v_p|lY=LKau?M!sruk}PUV0-p`?$YR-K+PWkw9KRTK^(F?fLjw3Ho1 zvtoqoV##Q$xaw=63OatWZ^MgVlP;OU5H90W8w7Z&wTD_(_{)(ydWL;N{;iyU2ar+b z3Dof#Ri88+2(2?emhQ@%73gY+@+qF?oE|S`|0)kWGifkm|HXK|FGj&;=V6k1TWkEy zeN(WzEn@liH{Tfq>a8Tl642W?rsJDSu=^81{lPfCGXBGxuC^M(SC&XO2kNEaA<2V; z4)ZP#OI@u))8y2fx6BLa#22=;iIqkw5E9k4s%CMH?}y9LfBcY3!iTyterzDqRzdt| zd}&uk--ZG}y5H05Ym~6KRb{Jc2bIbMPZb!JIR81lxf&{$(xx~1w&#{nWPHSx-8zl` zCKIl<*?29j_9Z)oZW$Y3LG&UK;)GaI9(rXH92-M_U_%@kLBP?p`K#(GZM0td9k zJ;jBvp^%JJMT3KD!ER0%06-0Y6}5J8)A_MXDu9&%7D19giSvt-;9Wc-3Nd!5Ussa1 z9ZZ2C6kmC=kJV)nf)wuEL}e;*FQrmq9Y9$PL9WRa9Fa)MWfH|Eht2)XiIWEw;bgrb z@+T60hbjV2j~PW~*H610pUc$Vi0!InmJp4|CJ-pBl*mY5IOHG!tj@!EWZ{NAsH33N z+^1}m=f%uqqPYsvu69MkY1&Y^I4F(t{P&+>I-%e80-0LfU*mdLm~N@0Cs3zT6%0h9 zypQV+l1SP~2zh#QO^a8Rc2Vw_j$oU!3{946HA8;sMtVPEB3i97d2w;{`044;wQ zHH@k084BDl(%2YVf90xsMs1wyFJk<1Na$nc{5wR|%)akzCz!v!C^qrnVU0rHhxGfe zac?Qf9NOHrmm+kMrNq>A7Qw%AKIDvMZ~yU(82nl_K4v%?wbX6VG2+JIn&JHrZYF3h zMv111@Z_Y%I$$5Z+>OK~#L&vpe61n_q0VUyIsR~Z+d_`c<}Et&QoHLpmOE-nlgCW= zNr7NR6=edDcOuR=1>064g>?uk)-_KvFn{fx!(-^-gzouP>+E#dWGB|I3KP$FG8(qHtLQKQ>lk}j>jIk>NO z4-B2@iDMiR(q+pmVpmi`68FkPRj;T8chR*S`nJ~`6PbK%6B3t1&gT0u^ati(b61Wv zJoA&U1IN3@;hhLHH1#e1dE8rEc=AK(GJmx2rz|aUPG8{bwD9`2lJSe>%j4ThLL3-I zWo){|A1I)rFA{eV4Mt3&>48N|g1&cVm+1Z>Q53vq@=|~+s(w07r=xWln%1wxzA$6m z6GN9CbFuG41A%Fe?TzE3etS~~$RaeniD^%{rb_mznI}I!zx>D212`#u zn(PU0+Yvy}sVn_U4T=!k=Tco}c~2~$%sv6iv_D3H(R7&FTU^6W=-cmN1mkjD$9WXl zWPH2KMRbOPQ1+5&RJOhJiLA9mNaf+{?%{q^Zo@R>6h8$X39WRWJIds;Xh z-OWkVq?Az%n%@Ni*uF0VaKHFAJ#UibPUijBLc*{duxDl_rm>b&kWsOfDH+5oS{p1S|EF?&T` zm}(eJ#KFVl$AJTv+RL$B{+JO^K6~7?yl+ZlrNBD!b94IfMuzg{pOx!3r8GA64{q<} z=!{9rU+jEwzrP!#4cYCU$OH?2V5F+j1=$-;mQNCz&A`MLSUKs@< z#zRt==^6c;C}(dtfAU}^LbWI$iUEaz1Qj2{tdtT1ET0*ageIM-WT+I)rv>(o9?DK* zkdL#k4ecF&5ak?-#>51I`B7sCfFrmfCThUww((Gcl9*nBE2h~A`;X248gMrN$`%*( zt0*`ZIsf+%kph&cTFk7La{3C{uTMH7HPu-Y(j*Jgzw@1va6YBIF^t*=Wv4D!Iewdr zOsWC^C}R!N8fmGjsOfV(%6`O0>=VQW_i~lY4SamQsyMrzd*wgmuPUaRFEyu9dDw4g z)wz7?>-D#1`s=&c+sOCdx(|12HFUNO;!>jrF3;IZ;^w4XRrnbFta5bEBOJ$Dlvdbi zBR|#s>6|B(jr)v|lE`KZ)Qy}Y355#|dUWLZayd^5eH7f=b$fZJN49UGqt*R&4p*7< z_=-C-z0Z;wXxJaIrNC_b3nt$hJ9I2j<*J)kZ`?W&1#PPSed46h&FVq8qC(7 z^+H7bdEo1Pv+HYvKV%^}9giqQbF??QdQu+9{VZm4>6cI? zfQ&uP{s!bPvZ~H>e`|gg?{V^_ER7vu?sQZu$Ojgj9qMri03V|5JE>eZNzrwGZn zA-hGOmA3kti-~y%C-$gs_X%lc?4#3X?r5ngicl-Z)*@x5qE&g7;%6}&n2bdpsqYVo zdMjDD+f`oha76Eucn96P6(mx4Ev__(wHS7}y>>Xh3Uf};Rz%%Fr#g571Au{@(IJ0q z0W>MvJ852U`a;qgJrv)}o#r~7ZOO@18o!+NQ0cULc2)}JuwP(*xiLAlfeEgAwWSqT z+nF%SW{b(MuQ$JQwUL-e8XR)%l}=R!Yj{TMr|LkJRAK@dwGTklHTz};Z9o4V&@4cw z)JpSso4zQO!$Go`-%Z(xSygo2d%U_hIJ-XcG!Ci43ScUt|Mp@9j%6>c(|}GKYR#li zPS1+t#HIh-st_??Kk;+sJf~299&<%ZYLzZ*0~4Qb*@_M0opFs^Kb~!1p`Gl{I=WB3 zrSmgNPv7i2^Q~9jP5WpS7KKSy%FMD*u^W1cL(*7Z*?1*Zs=xki9oxf%4*jrl@#z0^ zs_pb1z!nu98PeqlX2TRM zY{a};T8_e`68>Y-{EbrZ>!i4`AdS@Mil z7$p&bhVDAR*09BVpv23s^*UjPoplf-C<9OK?!+TN)eKsNkWJX3*nM-Pp%u`bRLB3! zhssBR_8NIP{00auc8QeZ5J5-dIfKEqWX_+u?{DjY`d9?Ln;qUN8V07JmVN)FodCid zMxm1&zMnV6cEPRmcF&MW`(5(1nA&m7zy*yO4kIBt7FzM=NaUn<(hx<5yQ!JbH%#q( zmVO>Y%Jwsm_KMy;^5Fg}@(ztQat25)r%R?#x>evVz~<;(i$9Q~NAR50PHm3hv~x2A z`Jwxd5pB?l&3BTlvCW&#bBcF%e-P5cC|MPv4Ea16Z?F2cN1xvYirM^c4deg5G0z_w zr0SLln;vLhutyXrIQmOdyV{0kR@TtV04h*XjdR};d`i|Cbdo+8_!yWqDYn%#kcz%w zhaVfKrtNX!)cfPDSRtn4L4X3`Sbht+H+4v(88%(1MnS-9Z1_KxUcm{KuGy|i0MG~t z)^4E^c%U>vvV!uxWXscPu}nYjI}6#W3trt9*cxMqN4muFrnrX%Zm1U>$erY3Nv7#xO;aLCf>79_NQf_pd24Ore>5>n-o{*Yfgu0tM(V$cDIa}X<<+u|#=-f&FF#?W?&&(cN zOyD>=ydcmxBKoKd;_=_=1KlsBM@_D3Tv~*Ty-CW_egN5-+8;j~oWHkyUG3Z0zkORb z8gdX;s%kWAg&XnJ86}Vl45b0V_ zJ^Kti6q6LzS&vcOa5xV@DECVbHiki*g0y}D5Cu(^QH-o)A!#l)@kdVJLh0ElcPR?w z-H5km^Gxy0+@QtcX5#@BL71CDr+Fp0GY&1L44FP$(h-d=rpBk@gcOlN4fSaw(fimMTLNubzL^*Nu-nrm9s2-CC!4?d zAy3rd_`@`3BnP%=y~p~cd0gGA*WwnaijC>cq363dxbv@|@&^Exj4(Tq#NM%Na0!ki zsFBc&$H)wmUC>0XWlG55M#f9)4X;>9;@c(uO&s;5n%e9$%(9JCT>Kj%Ts`7Qb+3{4 zzJmTjnChYoVHeqj^P(M_dBgpeK;ztosU)jicH_t}srZr9Y$l*^625`*oZRby91$hh z1u+%#5F4VH%oUFp!!4*nFDF#gi*+TO5tNw}ljGh>M;g7jf)Ev2e&(1kF@PtGRN!oe$ffGp0;r(istP zG_cCzZII%qdLX?H??wPI-e(0)pRbTemcYhoIVuE>f_&!Ok1FT*orE2s5w9)(jT>p> zX)#u&)Z{&!ObvA2$e{d3=hD{{FBNgG5K<4NCNz zRw^Q_&QoNDHY6yDPQ!Mh4?d$1nUjRv^EIrrv7)YV5hGUzb}r^X3Jxx}j^OMU>(!^G?5QqzBo` zE6PhVGh4Crm!?TzE~5jS^p4}y6*=SUd!YLB@tX7-;TsS!0==Z9iM#D4H4fC$mm8{p5^7<3g z&G-~%b#uxKMe?7E6aA#aD1}K;`VcV^4-%6aoR;F~)+}QOs4!R6ci48`yx|$SkJV<> zKHit<9fq7x`~<*fTCpN$){^z$eb?%T@pz!tB-1~}-2%X+aYo0I1kQr|>|L4krA7?o zr^u%~*eb^QPm6m;qLJv;=w~^|Cz5$NAII(Q+oIAKo~7#o~!n*RHJrJ+QSQOi8lZq@mluTK)j4Ol}M=y+|+v zlNu1!LEV#+rn5@b7LX!FA+2adW17d2X+l*hzFp+9J*cO~N#`nwr-*#As!5v)NZLl_ zR0s$`-tv0Pp`|H`cZDFQ9va@2P$agBZfB0gS$vJLz!9{#(Vm>(NQZoYSbw-C{w@8~ zj!prm{8*4J{px3ypch$_C!PbV%=d!dYH|$vjnV`B!AoJ^el}I&!kWQo`m$x)liC+5 zyvJ42gPz{(9tHaHM#_W?NMwU0oGcPO(CCIH;y>WvpsOU!;|6?o#d`MO@Z4ZJ=_Jho z%^&(>7H$(rq~c2&`nn>bzEueu{*OL0E{uf54484O!n+i(uFb{n!7huMRV24fNrnd& zfJi=RKnU%bR^?ozXIgTXa(!3?C=iCSXrsIa(K-vSldpf$6Q(ubs^q*4&b4|pdu?_} zXOd+y+$35{L!S-`>*JJLLbB6S%d{XfP%pUYsTd|_hpGC8hm{SkpYibs=?d=pQBSC( z`u=&m7Vy>JBLl{kR8i29Q$cm+Wj?d`alDk`w^YBVLcsw*U4A$MC>>(rF=w>^k}4A6 z^dK?}0~^zDnf+jnSQA4-GJ-6X!Zn_&@n*sw2Ut;tftIli?m2U+P1It;I0?9zh$5rz z9IZW1*{?~+$}#%dZsZeBMN6g4Az!mM_^(J<4&1?sfRdWWr38mGA1fYf3Vb9pD6A_I zd8^svBV*?URR*u1xtXGRjyZqzK+1-0XO)YxF)vQEIFTdmuMC8CECBpMn(h|dSbxN) zi~IH!65{O_SPn;$P(roL;f+1$V|?cXHuA6a==pcc=N6|wj7-r+u2t5t6EdG)JFri( zC$dVsN4g9mX2iKfeuc^?BotylIUv`I2dDoKnw~2J3{A3EB?u*|Xj|#xc}`!bD13I& z`r0+l&UcmT3v2h6jZx6#YfK|$-v(5S_$u9v=Vctya(irWip6H0w?-ycoxt|$D1qb_ z{HrHk6Pdr(zA8zPrfQ_lwna@{FZxb%W&UH&FE~g&Sowqx-&*No`Qv9u%Z7&QspTOH z)+VyEfu70T1Q|^j8zVbo%<}a1qk-glVcl!nms1a;*EmJ!H3dI)<;#X%K6?;);y;q6 zEZ7ModeuSG?)+)Jc1g<;m|zD@%3Emd?NczN!By6k#bIa0(T}+>S%51pFsahMk{oV) zHXq*}BC7s!B2vD=YfeXa;6o@)`NLyUo;LecKQ+(eD}_olN5j>LT~kgHBL2V4@13ut zz^Fv}0x84o?!%wNx}8wKJ%8?f-aYc8W@Jhyo#Y@V3@*IEFNm~T)Ow0mq0>J^ zw^Zlp(ej^x<5cy)@HZ)DiGIjz8FvrgqZwJQ7$=5~vW*8i(7K>=ZoKPP%d68f?*AI{ z)K1p@p#oD)+b2%r%S`}37GazuI;hrZb?hH|E&xR5>Qc3mm^;9#AHl^8t}fc5HPS=a zi4(kq(fu_e8K-k!R%dMd9D?rr;;R5|E!;u#(UAAO`nr^RM4{P1%T`!03N|i-!p{*c z={(U6u=AC|J0CitGK{p2_#Y33N}&TPm3!5hwNQM$rXre9c#k`y$fN8m=5*Q>`gABm z)J_Yz=io*`I1N6Q?3pvWRUfBW%jM~Lnxaw$w-Q}3?cq-Q?o##B(NI1ye(*DkYnN{j zAc$3IY1kRcyNiYMH9QXtrl?mkASXd7SvAQfLH`sAII&`ytH!NMxE*Qwdb}`fIgHM& zuHd0eC7Mgc8g~$gq2XEH0n#M{|9Dk@_m4gMaKaX4sgqK|?VnXYB+KYa0vOv~IBbqd zN}IdYex7p^Jwn})Ez!tQ)D%i#{q4i&sWnzy7vR#QeXY~ zB_pYZKyw*Mdrf^G z7cf#{-UrvdUvn~Vx{<^8J8LI1CRAC60~_Pxl?j!@ft#yt6Zc)Qwd-$y<4LI`5cveD zx>IQs`R8M4Atf;&R5Vvm731uDY(J+~1G%?>cgFMR?gA{KRVpJ-2s?&B=t<%0swQtH znQ`&v(spYar_5wQvW(Rd4pewjQmcyUR~DLi0ayg`0YOZbymk?y>CJ;{!5?WMERP0wGH?TC~W&({d7L3Ep8A3z68x% zEfj4f+Lwgl`S5}u=C!(?byG6CaM6HE=NWBK-zfhpO|zcK%U7fa3b`JIsBaC_tulhY zB?_G01=OWq2jz({yF?lMBGNFo z3V9h<$cJ4<;r1Y&ofvj@S9f#8X>HZ0x<)BONF+S!RAw%Pif}{jnsL z2Plpn77BLXe=;(Yl2A=-(cb~kp$xAm{(*1?C-7zCIHAzp-lb*=4o|omDEG;!lHj68 z-$R?;@RNVmhEKoQ_Sl;-vVV@@X%FBmz{>UQrqb|P%0s*tiqiSR0&Ki}IxKthD9Ef1 z{dowPDEGR=%lqC;AuzR+Xtp`l1lUF$d7v{X6mw><|K)VMnho`$AMKivdX{+(p7}8 z#EQcd^-u0)xJaicEyGxD_*h zc+C!K6QL8XzHyF?<=2>4FyRt?8oj6P1~XFK|o!3j>=JLmv6f!4YJQFy+zkH-W%BC&; z*hQoE;}o%u|LVgl`_i_+N_kX6{X|7FJSPsCnwU(Z`E3HKgx-=qB79^JsrSI?;H6C_%!)~RA{47(D^@^W|fGmX{FPlbquT_iKO z^{7mDLrdm>yBlq{Pmiy~%|td+va-7PJ!+N9(P|S7jHG=0CE3rMKwTgt z<8c&<@LQDJ?bY)CEO)7vPD;pvM))n>pQBk%kd5zI_xz}JIqjXZcbuh!#towy0a*U114!h97qTL#jVU;%v)n`mj+w1KPPrWW z_Eb5$lre6wr09`@dR&-{A%v^=Wz+E#dx+BEJtG(Jje)1ksu zJ%g{0*iga~GNKecb1UNvt7vT{xvm@eCp}02kuSOAG4Iq43T1g$c;N%90V4!J>MZZ` zk_gJ`waBS;R1EIYvTgUy7yh~#hUg_C;fJB%GYp20kE>Dt5!UArir;CJWQun4V_TaE zO9Fn@gJ)Crlt&_rLg{LJUMt3OIkQ&KCr0UOO5L9S*bXD5Fhc+N`qR|(P+rMRsKLTg zn{$>;9iOk{)AxH^o34Rs3a~O0BNb10J_ZpwD%@kh&R!{bDV)KV8XU-GwoI3?8C<3e zW$g{J>gh+M87*OznW5eQOW&)|pUX^V;p&=l5d>#ctkE)EnKix)z~<0%!5aA6U+QYj zxY`h;`sji$g6%>ljC6`WbSzdx3&r>c!WDqPr%UP>`n8IrwCpxKQNn0wp;c4|3IgyN z@%7A-9AQYqbP?RsqnLU#Wvvt!=FJAsfjwSQE`c@D82;6l4B4qEEhTY<8cveaajw$s zwM=D&Fx&8+jUD{P8SRl^8N4;^_+pO^yHUw>O0T0%R>VnFBhR6A_P7h{clK|?9kGrB zoJE!+frkBBIJBPdN{`Gi+fj&thQ%D`40kjPTJg?JOupSrwuJi}G^8?^oJ$i9t$8{6 zsVP41hPy<8^6exmr83B<6pIuKx-}`s-6y(}0J_t8iKUJshGw)2y_ndJucF`|95Uw9 z6HiO6%7RWNod;KSR%v8h+2qw%Ngi7RO$=4P)>KjL3YXGe~w{9 z^BTm|`EmWZ9bPAfzeqWiKJHDH1UK+vKjAa(?Mmy)QX>htNFu1TD?qSxpILE!HxM+{ zI676d^vqN`MK`nJ%iY>$=izj<^E9j3a()QgmvA}5)&S-bg*1yoS>N`oH?MS7a!HlZ zVL08i4HW3IxXq~brXy80shXUBlGCQgT$)>D>GBhLKbDMaNRtq;HtT|YQcGG5yS#c_ zUJm*5Ytbg>Bh5d(;HG31od58@Spd<-bDjxkQxR?{b5|GxQ}@z`vdQ{TRmOV^ieh-0 z%4$0Q&!oz+R$--T@!DgGB3cvk-+R8GCokKoAA8 zi|V8o}?x&af~E8P;MwwJAt zuT1X)^vRfgCgk2Q7c_jVVxvE6Dor_x9Hq`9CN`;G`5|iXJj=@TkHr>fR0k8p&>TQjPGCAV8eZ2MCmZ7{hwPuCW$otHhdk1@>?M` zm96d4_x;H5FSZ9l5Z+Mq!zQWaWNYR$9&@%yAVS%093Mj(CP86xQJ{XySXPH3+f!}41#MES)DR+WWbby8w9uSdTn2a5#)np!ajM8NEHm24Td4JxJTQS*8rWkR3~kBLDo5;Tk<~WTBaHMp z$9}PsSj}$97~j%jkaU z1=^&~Zy00bwnyuV=FVQ`Z zz1{~gi=I5qSc&z$ANA$zTR-}gD$t~Ll83xo=|(Q-s&-eMtk8YbQ&APZ%J|wWruAr0 z^=ryydFoT`VY9J-0)!?fb8teTSZ48l&WnZG!^8##zowD-CXZTM%PV|*DBuvm8}d~6 zj`b!7-;sk-CKQi~JlsE^Z(Em>dx=&z%jngU43~$(_b3wLg8YnFm%!wUW99bn+yZ$s zb>Xt*R7I#0fKrtE?_8nQ-}g{(i9DaxNDN9R;4}J0W!LxMjnN5@u;M&}a`xN|4WfRR z`A$GhXYP?A$cgIlNS;Bx&$v7=`58U9&Fnf*FZ<8W598P1Zy=rP$_y;DCjelY3S*Rd zpzhDUJRk{;2zH@jC14*<4dGPS$DG1#p?-+?u{oa*XkyM5*McD;kJ}Ofl-Aa9DA|7% zJk`LD8f(FjQe1wE{qeAS%rUvojg@{G3Zsxf4dmiIY=1CUz(Ip8}87l&)=JIGEGtIy@ zG7SLyvC_Odd026n$Eam3)Mj;r?VyeVd= zfU-oK$XI;tRLqcn`}qmL!E>Unp|D0FMcDX8q67X`L;a9Q{h8La#8C&48k;)eeEb(i}D=3ZcW(noGiImvNkf zuJhZh*CR!_I{AZ#hQ4;Xy2?-|B0LhJcm`bch642jzIZ~e31!TviUPSwVI^K1tbV3! z=QJFp-TDWSb=Kcoa5=T_uCLilm#*B5e7omAy!~KJdQI#n6{))^bvi}VT0uBmI-FEz zl5{_Dag57z6O6p){i~!(YCuw^zY~m-cnr4F38MFZ9<{lQCq4^ciPJ3ydli3r^XPze zkGwE8C(ZvR(oq25Lr3jLg>?$JN~75>6^rw53-H;nN?b(Gxuv9|G-=!OU}mkiI*Q_; zlOr(E;D7@Eddba@4Mi&}(gspmUjkJnfT@~Q0W$$gI9(u4c+sVM=GCES-#D+893HDU zjo_vjOLw$?`#;-!gT^)4=Iu;Jg;*Q79QHJ;e&w8wY<7iLIoh`@7ip%eRJV;Nnloy%xr1xdRM6!I- z&t;5vXwdlX!0VI0wY>#EAgbtDmX|vH`$*32l$@X{>b;L0s@K#>vcL+(9tfg`VD)tt zo}nw3_@;gR9G6X{0!?=5Sm*00>>OLNAI6`LVV?F{tA;s>jf+A~x3t-5rY2ahb#hbs z{$Sp~ynnzw!HH(5IqK>?ZN%&Lh1MH_jg^Dynq_16pRWzIJ?-P{nNS0!rY26j--*c)g|(Jk zD`QH;O1mW{#qL|1w|T<6uD`i$y{g3!#mncd=8R|v5dm-jMuOWRFdKzShKj6MM(@YB z=#VraqO2~yv^a)X4motw=?-KdqQswQh-@jbF%|k-c4QVj5+oAv8zl<#`8dqEF%^G* zCthTeOKp9(T?io|;mH#&5juz-71(yE-_481uuc>N=`qIMfI29ZE0)X|XJ2h8yCSph zfDrHo1!g7bP*RhnIOU%1vpoi}8JgceO_r%hGWx8~(HFW!=5BpZmq=cBIWOA%sCzym zMDB8=uAoAT=v5Tks3rE#iN5bod0xQRG!CE#2p#@-pAS|hi)_DmY-yQJd}AR?;LRe2 zj$ENApz(vhu$$Ij74wLscwc0>5p}1kA1lDZSet&ehR6NE$E2b9Z2Xvi>3h zavM;SdazK+W%EM&y6cg{P=#u9so5})wb_!IRDyU|W@d{`-3+iOFB`}mM!q5FICZMV zYtQ~J2wSJTeKoFpQkip_xFG{56IGRlkqWIVy>^^oFngPYMlxfu@x&otWQup+zUb>T zFWh7uG_Hf#C!0rS4{|=YtMW4w9_j5jX_RQxDwgN4^ZfHvK67PX%sJeKm+rzjQLz{^ zHY)+>M5N}9w*ZKeh`=;3AWNgx+=z+RL0{DbmovBUZz?pf?{1EM#e>AOvPDv(CDDLg zv25%|CEGY{y#UEBQzV6{%UJt+Zf{E_Aoh?{vB_uFU=#U3l~lP`b1BS`-8B1__~2NX z04u18Wc+8d;)az$4>S;biG>IrCm7N>=Af(nCT(HX6#J7B23r@cR%y|NqYa6O{mop< zz1lUF(}0Hn4o$B|O=5_^7gh!H4+3cTzQkf78?eIBe(q+1tee~uVmI|4WYEe|Oh!w` zpBUm-c8g|JJg$6#y#%X5_WF4YnspodDPYeChvIy^?}2?W9?pgMxIbVLq>2$V>Y;?# z2$uB^-%n=Ht%_jZ-)xAQzr-VG(2xNu5eI@?|HC zFuofdLo+%S>0hb-1`yslD?gX`)0nW8ixji-n#e|!MQ(}AAs9dI%9U#`pD;4Lsl7;2;X;+>Eq-gFS0xrKj|-DvayXjYKg^@eh51!9I?*t_nv> zg$B*&lB4cfWnDRW33F#j)iQ_RniJ2B*w0p*KG#a|k-D(_@+BhJm=+JQ zRz6KsQlfuxY4VR(W>UX4?lY&34p=x}@sxb{6)(&B$C;gk`23NCeSK_<;KO3ZYB<~x z$eeZH5>0IM@ZgMTQUVIMyq*2!JLimc21px>Kh?CAWNRoIT5bhXGcS=bkyRVCf7!QL zbz^L8d?2F7fnNgN@=QGT&P-OM?X_Wi1g(-bWx4m+z=5%#9F3aZ!T?aRikKoS!NI+r86R z5`v^PI9lAHY&(h z-q{*ZMb+1Nk1h%O^?%xEl8@w~%WlS$oi%~0593{k1Mq7|J09x5&j=%m0wlZdrMRfr zu_O$8R5H>U14!i_(;=uIY~HZpGc{k~lu`FQI0qML*m!HBK3ypvN{I9v~Eg!Quv7(7N6gxN)YtdjOaw>1)^`Z&` zE?kkfMP5MXMd#T67``!;7ux1&{0@Nk&Yo85nBF&qn17H9_(HYZ}@Wtq;cgld4_T$I%y|C*M?z zU z@;P%t)5!}3Gmw%J4$1cb;^X6c?YtB)+2L8?z&*9wL`zUvN{pm@#+}K{bxm8ghDFSm zxf2bFDN_zn_NM21sEp5Dk7rJ8(`60pDficw0U$pxKRpNs;zsuBrhNV}vkx8UsQ&r$ z89Z|@P0U!w*}dhI7uGV_AV{ft^PGoLp^jCU^ZxH$@IA0VmddF!5}%%gz#K;s9x)|=|jDHjcOBj&9tAh&0Pii3k1A@wnQQrk+yGOw@+hvbSjGTeCJmjsBk?zTx+tN)8mFo$KmZR>u&Nfd8j#|UMA-kF>-Qt*H?+stm{OwZW~I0 zsaOqkzKANJPI!jLMa!!yR9o22?B0i2d%3=D-EZgy#^>1(bbunuo5Eq;nM{(O1cxGB zto?jX?IwV!!pb?Ub4la0NH*N56xm3)2p=F8hr#-yByx(kVsw_1 z==R@x96@}%#Dxe>z6@9Ca|i22L<)_g={J}SNRkK{N^DN1fx7{FK_?bb)Eh*PM8_q2 z6}9}Scq{7w0uRolwD^tL6ovH`r5L&WV&Qev$PVukjBopvU+`(%7xqQFpCRt>AT3O9 zl*k$o+c~?L8w|=X$n9ASPc`kMuFW-aYU~Mw_tM=2*#rW-wA!b`rp&;=Vit5v_+QVX z6zr_K7(rC3+e0GIe&;r6JR~K|n8X$S78oo*>g^Lsz)C=!M0(tVoZn@3)s~LlNKG|w z%?NSxRUqCjPy@__G-PPt+8-y!p=oMWV6vkzToD^QbeBCETh}`#;-#)AFK+~}h>Lt5 znt{xch#Vl^?fT=LVzn3wQV0N?mmdLzE2hT?#h5v>0`?#V><{A8@P_6t7~I#Tml*sD z+^ZTWK33{{-!xyG>!`Co-wcpd5{|m#{$AEgXdO=+@q11HAl)j|=516DK7rV?YZD_+ zVP5n^pT}*-ufjT5$hcwC40b6nf@@486C41@Lo`Q>uk;T_2rx6UTi?1A`|62Kr&OeX zAt4N0TS?qS<#$q%Q}UOfY1$f(fH!V{VibT2@y6C%*=G^0X6F5r-k{^_3cpgzY#PiY z0uo}k$Zt?(c2DBCw6tXu4(yQ>(R*;6r)4_1@&vev+(>chiX#8Iw>@QN;mP75ws|7v%Z2YRTUD8r(38a2+&3iQo zsK<5iY4FuERC~c6rMb#KjfV*OSP7EqW>O~Jv*g)^>~>~NQxwz&6vRsQYEDR6_i9Ku zMjLcDx!U-4IE6%Xgtw_(kvyh8iw!pn_wB>er~!R_<*B>7!Q#HX`$172eh#N zE{ZGM_@m(Iz*n$;rj|;89s0M{BmJjiunj9U%BdmCz^6}ZIu)i>Q-9J&Qk4rn5@h%P zlte&c#AKsfc6U?a{<>ocQQhAajQMepch_ay^3YnHkTH zI*j>A6$;3i0Cw%0$$7ty(>YvY+UJ4_f0WxApfj3E0EUpikYjLZcK0cSAW}g-(d$!_ zGjU~k`o3g@5}gf%iKpI+*tkD@T0B1b+c+IEYegFwc3ik|odC{N4R?g?OVW-zo3sAY z|K9jh92Y2@8(;2tRaLx-{e+Pt0=M44L-y8w@-}fx&G-8~-|DUZ_;dqv-B?i^cT=P< zQ`xZSP0bEv(z&J;Q%UNu8i@prtXLwo^BJhEQK^o;fg>c_($D3_Qykwh z-UEfTBoQXt(Q8-ihB zey?R`kwl>hxF`4|Cfb&2_4KHA_x-H#?^Kh+Ep{n)m5VI_mS{p>B<^tAFt#KSvvf&^ zPSO~R)oI}6L##=;rp>fmWChgkNv^oNRt%laoEnmc90$3PI>5)Thmt*2Gmqt`?}S?m z?>v8~=pa>y1}Qnuey-;eD@0#omRfV|6T}JY82^*_ULj0)TBQy6bM&x(JWnI@C)0ZN z5BAw8w|rXC0S0g4-w?G%fj+J|hDW?<@sFL|bUB$Dw&`Cnl$>f>k3#9x=e20PV(*ev5;NvFpwF(p#JjP1y>XeE>>B3L;9mW8O->2UaAEHUg-4gyMi!{6y;F)xX5pIB)hoe72sFjV6WgEm097IB~wA5Dif&pK)mACO9IdwYC%| zBTi4`RG%^i@ntp}e#}ORt!9^x$bq$&x(QMP0EDuF3P~!tCykt?!Nmabhhj}+$xTre zi4SL3tP>}JqwUk+w{L1?(m-2|C{FlXC6G-wx)8$IjO$Bq{fF7y+z7@NHgXXiA+C_;F&j=% zEkdEGlOdVS2(&0?S6QG=W+UX|bEk>^j(=WYH#_LRqE)t0|Gh6c{%$||^P9QRrQMCK z|2^zR67|o>h(-fAnL9)2p-EKKoD4H}+IKIS+_X+`vqZ^cQB+FtQnXw?b7bf=wxOK+ z3=L^#^~~PZK;nGU64Y7}(t^MY)}x~g$;mzS7pgq0M@>h;t%!|DDF|6_@@O;Do3~(V z{sgnt772c)%lpX2u(ag0aXZ#ho<5Rf_-)76}sNMoKmHqeT-2qnTJ=Ndz2ize<>O4^4G zQ%n82v!^`k|0OlmI4O@v@8njld}Tt0B2KEWvOJ{cmXMnAxoXd4cJG#szT@74uB`XS zD}kjbzSPEDokuS|L5Q_=H9vL_JsedLY=;Zq{(9Y1O~zSEt%ozIX6Iq^$TS1zS3&+r8-~Y0l>c&nAXK zWPi-x$$RfkeAsnNc{ZHW&2?s-EofTiq8#|CLx~8j6duZwsjK%QWy0im7`G{Hh_4kn zLwKco>iH<~6!IvkdMc1WpkWYXK0uA&Q1%qL`#Zsv16v6Mgm)v-?Tv#l3{FrCYL z!8a?*DBvQ3TJxx6GJ7CAv~UinQkVXocdcT<3m_=FGv%|G#7nY^K~{fQ`BneTA(mLl z^u3qJaf|mp(dEnJh{yYGz;%;7JSqx`EFt`6YLpHLl^*?fl%7Q_BAkddX?(lQu?mGrmPxP1|DZt)d#l7+tcrQJ=oq``vZ<_Hsa+qEmV>q1DB33}!cG@>`i; zEFC^OJ1mj0uq}8lVL4{-x$d*qPTIJSVB-Lg$bM<5tgf%w)6^rmYo&}q)I@<(ZA`~4 zG#qed5kp|Upc8GYOvzBCMKC(lz(#o>FL{?jBRQhQMQ+J_oQxe6mRD+mPZIL`$uju6 zO9tIOylhmF+Vwn9Pro@IkBBK9jfsxJjbNjvm|_Rp@PW>-i(u6$T+!P>ULW>z>)OHW z&l4<(`iA;D-{1c=Q7fdGSqC+X4-v&4}6(h-|&2_FzX2-hOZpA zG!K)5w4j$J&};Q+IlQMNt7+V&{IjD1+$pa};)at=?pOc%@Hr}S*~Pniqca{IakDNE zPfCt1hyWvi|6>ua`SHopr!oZtc0ykH#DU9Ah&oe@<(Q#YB`qeZWS|vw%~z5a5~=`s z_7~V=t48t%zHYgm(zh5(pO%;xQdh2<7l@$yySBzHYsB_{huhYgpNa~WkBX%Y^Zw}D zr@fF}nEo|^Jq&$iQ#`e1UTH2XiTA24Ya6jl_p&uZmm4RxDDenJ ziQ>sh+@N;90e8cB*A8rBgtBrm1P@5i%2SX>^G;vy-dYdXg31axs8G9h9!L|(j`0KY zHA?P(-ld=;nH9l*)PREE`JX(x zC;ahLeU;8goqI*NtY3TU$Ajw|uru5xiEmP^%MQ12#VQ{lrOnwQ_;RN_?p)VPPoO|N z&dN)YWG2Hro4=W9;GW(`KoL1vFE-UtzJMS(oJ6LDObAZv>~88fI(BRFa`W*qZQy zOk*mYXl_{qhQJr8Y}+Q0{Ys5ZCO~6BG?~#RRG_pP@(t{s~^rm$F% ztd65FsVPHkIqjq3ZDszkUUZ2?=}!00TYv$=!G+CqsgUkLAQEXP9W?@GMEBgVnaaO| z_{^U$I#ubSaB7sT{`k2uh6@KfZ>8ek)sqZiF&@NOJKLPE$oK{xk7EcZ1xM_D;_L5P z91$N37~0P#TlZ|JDu!&&(q6^(RTkoN4wc-eovERG;Hy%y8y#d$hSQWGJ;P_h_hF+f zIAUuX#p5ri!EgUdZ!gXX@UqhIajYve&O=3nz*!%at3sU7bCG0e0KztiiSgF78|>*W z`F9vYN?JH~clX~WUmWDVRUR9Ro_*^p99L2K_2vL(J#^EImrT7rEGhMd?RC(xd<>=W z!wjy@6^X;;OMB*qWhkk+M1QK^m0#}{e#4bJdA2a@JTO|$+hpI>;b{-?9c23c(!f6O z#WE%$1utkR5v@_S#6{57ehobc!@u~-N_(9HT>+_ys}%&=XJZNKgF$OlRGqtcr9*F)=1oq_kyu%BGkudXMf z{wseRK~1p>dK}}^R{?!v-tXzHVdmWlSfOCYwI%>LJv{YWS2FpV_#j-;P`RM!biw88 z6IA>w$f_&u2Q4)5v11U2L31Fd*;%vMl-Ir=t{P)X0-EJ&o@2c}f zb$D__OI{gH7%c1Jg8ip}bS%-jqW+TwQCZYL*S3Swz}?ay*SGVm*~bl6MbSz7AGfeS zcZ=^jV0M2GwVDfAq{Gqxh3iX}4h{yL-+T00OM@-GG?E6>7`Yk;i3g~l3W%Ek1VB`X zbRC1lfHXTcb_Oz1s1-IYDX)_W)p4Y>ac$0h@k?5FA^&@1tOi;-@aB>bR zmjG-yRgnwMnsRz3%jX6f158PCBTh3#vN@oNPJuQ&9lrjD+KGn6Ngx!@TdD_)Dw~04 zDxttZE|ct*EJql~n-;K(OULhjJs%L+eT;A1KdIIbn$tjf{-$gBkoNcWEkoUwCH0Ws zWh_G6xWbMr5j`#&-CHr^CqLG)&2V{x%FnAAs`B|0x?$zZ6e@E-QiG5091;>SPwyXT z-N<)2J`)A}&W~^ZeKTRAzKl+D0pYoHhaGM!=X`lNeCkvCPFhDfSbOIVS^YBP62XP) z0_p$~9~7Wx|F(|l7u_)O(ogmK_MhPY|4-QAH>dyq1Ka;MoZuH;Y7%M%}-eNp^Ax;-hrd;u2&h3B9pET5vwJB+LcnK66?h>{RFi{NU=OM8>r|* z>D8W{Vx&~Zw@$(meGz{ad?t@B>8jP#y1FKa2LN*4N?Ws~( z*d_n24#ZzN;ko)u(98E{v)bdU)`q!E7vCHnBn{N)ci7GI-sIeBn^;idsP&dL$cc#R z?wd0kOqYCTcRkuok*d-WseL%}>9zlP=3d*(NgXv(Xj2BX;}R#6J1VRu#=o3j6q`br z*v|M^z4|lM%sz*!8soZ*zuMcXD3)*)+%G< z>ZG~1mAT4*bfSCD=y#7w9NiQL*62CR8I(+Km0_U0(G*i7kIJER12=A_Rs-eH&=5R+ zXs>2h{75QheCFGc){nL1czlsxEgXM4m08~?~vUpyFT_VDWhd=88>q2Le+u0CY*T!r)tvMo9E2-N7@nMaUHLXc=bm2p-NHvuU4*0WBwtTgaR;qv) z)x&-YtUs?&2++tT|N*w=Y(tM2!RJz6`uM%-!d|B9L^NJO&Yqc6IuEP@V|P#|hX z5nMse3rp%z{Og5nNpqPl@M|7nUwC&bgi6!Y(|RQpyyZ%j`XjJ(k}SlhZZMlu^Phrg z=a2XVmy5m_F-7A)=`2rMsN1v_iN@s3x5@(9O0A#R;J2Cn4)%w#EUVE!WmUB%CI$da zQ*Ld}u2rW?G`T34>CxN;7lw(^vGe7V+#HFcqI-McDB9=U=atK zTxV?v0d>gYptZ9gB%7ksz<8~jZl5+t{4GQ5&1YT|1R4K-rMmBLuu!6WMs_8U)mZi% z-1#g0tJ2T&wNwpRe*l2IHdyhzA&hKHr(h(Scov|J>Kola43Gb(PS-mHm%%0Bi(^iKGP2SbG1Fnwg#I~RhM>pW0#j}T)iCv}?C_?ww!NXL^0@%A3c z-az4xpj^rxd9(s6s(3k2O>=1J6>DTJ8hnxs4P>7B0CWS~9~Pgi$9>-`E}2=dMIK5@ z5@thg0QEY`#Z7ejLNXjs8xp|4GK5YlAPr$ONa(#QcU{Ae%#EKU!oLZfrRzd)e}tGi<(P2%FuYKSl=ORefFnPGiHJP4-~ z2Zg9|hifdF#&I3nH<2dr7h!I+PRz}C>~Wt9e}HeZz}lJ6F^iDyK~P}B$3yIjqQ^|` zP@#4znS+=WRS^cUfgV}0G1#rkag7B#zOWNw_i* z2I^R*NQ8`z@Zd+N)+d+i5`2Ta4T!8NKl=_qJynxF@}s=Rm)?Y0bW~yu_+L687mftX zF+3>>3EVkJRO?|XgaI>91rl;g%MpVteyNlq3bl?Agih;UCj0>y8_&p`ah>-!3LX9l z-ab+}^mnoa@FsIuhZA89rB0J+&fZdQ??V${9>8l|n>I{&dss!Ivrq`@DMtygchHa|1yE0L&pp6Pc1(H{#?_G$e{o_~um1bh#ig z1Xc`_u%GwN{jLZ@b$Ir&fAih*XyP`wM-QI3UF0!47nxN}ZIs;mzLw|{X!>g!Zpa<1 zP#S_Tb(k^f_))8K-1=?p2Q;;XEZC9`t>x`~>&n*g7uAT25+H?Y0#kjTMNp}Y@!}ix zCR7FWgCuCkvpObuE+(flA|{aLC9gO5-FiGyOoM9@tn)~+le&uXE!#9Es`i;P6qi#Z`1|5J?pS0XBbmZ_KGkYwhA2Ob~5Ham4m=>w#MFId}k^d^JYqUgViDhD^;uKX1pFSWk;;U|Az zm$~SaCM;Dw);c{w$czIAPZ`ws6BvXWOen9;Dl-X~5{wS%zDR>%UTuN{6;)AGhsA;c zhVae-G}V0Lc(Zm4lL!ymSVRZNoSuw|&WqBaWM5Gq&WK+zp|*d?-{h)C>WmQ&5&nDW zpHPt&R2(=1bL9wUSKJv7#!yo5?3O$IZf-_6_|RB}%qsB0oxY!%Avn`4`J+pSykb@! zybY=Ba7g;N73Q7Vi*+8PoULTWOYW@byn$Cnl-KjcM^1_iy< zAaX>j1?J21^LBF?Q~&@0x`5XL=b`04Yh8c6iJN_;#)^$>nk$Qp|H-Z1WS~^v?n+{e zE3AQ5By_y!kUH(^qR8HSa%P01y`7LMI)@GVmWI4bi?%JeEP?Zy)QTck)*bRi1r zEr(056rwU>$H1^dG)xCE0t0KXRR0PXr_%S53dQi!dFGjams+(-8O_loMOW|0A(Dll zTNCFKuP=JCaqSyizrRW9V*Adn?}(Hv`bWIbJQkhM&IZ$VhvU`C{ows_u~A+u>yKoj zad{+I-V7Hwr13c|vQZK>3bR_5P%Z}@KCxid`49Dt7RlTGJ!Srzw}0D`PqbkQA5$A2 z+sBqLn1hh6m^34;83~eCD^p|PPI2b?pYd+u3=^1wU5FFZcrqB$MZ^ZJQ#@ju`of2E z%oruN!9)?;YCgb`0A<$r-DZ6ul>*^uxeO9gJUOAlOq?Q>;I5B2)~0(&NjoOy-?0-} zBfa_ZIti$WXsvEML{Ll&Id5rpyjaMGhy6=YR4-C(*j3~r#;dSxf*!tL?F;^G6DhQ0 ztCaIVI!dusZ3cfnB$`CD6VUha$MQb8T%Sla0|5Xm;RWz;DF3-8_KyZA4ls`uV^?=cD=0Z0zUci(^P%f$=!WbDBq!{?yVVDuct(*_JlVIvr{JgK` zp4;6KJVTt)Zp~1|RG!ItRDd9}Xx5m(rxr|xD4{gNIlCdLhIb@?r}wQqgl5(Z2r_D_ z9w3zSW5qyGOc^%J%g>`uV=Oh2j98~fm$Poix$Z6M-SJIbahZMAh$ojqOuh1=KBGZk z&%-5p&mfySSl+HHxlgFg-T#ffkIw8WHCuy}0#VWd!v>9wfFw{`yrS6kbx0&qnC^82 zE1~_7g8)m3Ij;dKd%AD^_qH?H(+XcvtQhRbYlfHixO<_WIRDN`*N^QyI61sr?DfXW zBAHl)e@_7K^VNx?%2t_Y$I+f*`z#dc=zt9BR1tvrCx!b<7XcsJn%7dImQDA?!?T=C zj+L@Z;!Y0yY*|xw%Cz)O5_tFsRCu+FJ%PC(z1}X(&u?98bdatBFt&P*Hb(IAY7Sba zF`972U2R&(+AZ~1xMJ8fSs^svX?sHhBi6Nou*}#-XjWW={z1Y7!05IBGG2nCh}T%^ z6>3~doIPMVaxnYtUK2|#nMggatz&A(n4>Ao9$0jejP7uTnJ!IL63|w^SY0{?XWPvN z7nyGwm?obtvpj3p$_BS(AlPIvdMzCBmfGS2!q<@%?h%X842N+@X(>Tu-VC|7CEN2V^YBWN*4>?#LJ3=QrPL)jav7SNs0^Q{iged-lo{&Is0# z=@|A5x24!xQeq@~4ZLu{a;41T$m%)D+fZHh2e2xNB z`^C)syH;xg45KK+U=c@pFO821Ubikaa+yK#l?7OFV6or8Pq>J1kCIedVhiF)EiVr%m(*RBh&Tu$>8E!$dDyj2S^krLYWu3od?bGwe z?ONB9V%#Bj8gu>7zPKzq8fSZ(#!VFTLMnReM z2>Hg9FI7ys>c48<^m{5IP~mra_sDk7sM_@1Rcm1d)Je|y?@oz-q% ztC;Xl2iwH6%_V%StbsLwqU}2xX5}~6;Ywb1d2638&jJ^`K5YN|cOAL)|2XhUw=?2M z+%{UhW?XCR=rMdLM^TDO~#Lr2!~hs@k9eZOaHtWhidY}acLZVA2zEWmuF@b zhuW4 zJr@b$bhCr*c!EcY3qHQuLY3|omHR+-v(M2?F3Hpg45=K%9sZWlCA9g$WivLsMODdR zjbyIK!~}yq#EE~$^9r>z&WXZ0j_H0e1WO^sB|M@^zEmHSrt#6APv~n)s9Ro^m&ejm zCq2#aX=zspQ-`ssaWZtZ5x2WWFx4ydRW|8S9*7RK!;+Hr<*ccVDY@^j z^$n_%=e98qbv6Kia~BhDm@|-#iVFU;uP9A%_5_qjYGGYH`IebnpimG|aiuu3sLW*Z z@V%T`V&gZHfiL#jn_A~TH?lspx(mD5z@74!SfF-@)8WzZ(345lr80w(DFOVnuwhe2(6`A6U|RoXLIC!HgxFMJ}t) zmJY`1gP9!k38_J3=Zv$dNGooJKZ%idz?+Wgm`L6c{f)&O1^y{4O7q9au}C9WB5}8A zg=KYFEfu=z#~~Y;meA$w6vT;>6yp;hYee!I-e|7sTL%oZMf%m!1MUL$`r%D|GfNFs z#Zo`0F_*#$85N}>{1}~VW^&fVtO#yA>GN4KXLj5C)LK|lP6koIzf9Nxe5xyv8t=^M z6Xj-VO)YMD(Q_QBtj`!d{n=LTCFP?^g(3$bK<`dOL{bf{U4X_N$jZ){f(O8EX;igrcTO0*DJW}t}Q@ZI4l=wSK zo@s=2Vp`m-MmqndR@MI3wTtrG(yn6krpU;gRRN2~Oos#g=gj@`tmKcCc0`*Bk}ie6 zYgs)Tx%>4-A7+*KT#es#H03QsE6(mg%~e2PBm5(5)DNp1B5rnrkOG=W%p5EMV%&YY zM8w$0P0tIs&-?IMV_z0km?O|I02%d({35iz?DNHeO~ZTAhZb9&@rHbja{=9mLP)??BqXfAE>Ns8IgZ0g4TD{R0)UO=?6coa&)rym5b$wf zE`IM5%K9?F$fs^NKJ`vrLoCYGl2TFzTx9FnRWJNM*MMc%V?Q*KmK?fqmXBNH zbAxAaUE~}cADe&uX^STn&e3w;H{2f8%$H2AjdP*`P2RzZO>roJ7iU}R@?D>%G#wZvVbaYUA1vkA&WB}+zJGK{(r5cI{-nrpZ3Kz z26`8pjmUW3XS&1!L@0rw%oRZUi#(Q1tt#a6d8OuQBzsb6i-0}f;exR3G-HKc)yl2^ zn8&;W-+=Ku#}@${*>RdbVGbEMz%y`SH*j_UWrX685cYNV0JuL`8JH)C4DI{vqLh9) z{~}l4pGn?{GH^_Q)pyk+F|YSqI5b4pc;j>i6%p_*MicIxlsK<>6sju9-c1rYws23Z z(>7I}?lixqW-yKj;KzA@U#kkQd3DIl76V?emZy80_+$ zRF}Mn$MYQD$D@0X8+=?JJXup7c-bd!Q^%Lhmg1&+y@8cQW}6T0{OXM?w>r%{+NfX0 zq-&a(Z8!-fqOsXn8QjuBVJOJ$ZH$y6E2!*J7 zKeFl6?c`-q5R_KI+Bz(`_MC)PMmbcW!co=Akv`92h>XYe$7-;xV%}*e2^P_ zP!yyV9yFr@x%QPUtX|+7VQ50q4Nt55{=0#&eTRjT`ROIGuDl@V1~+7tEP4<5 z7hVE798#JJk>)=#nzcc4`R@Q->U=cm8#dC9R4frx(0QR_!U>8JTwQ|Dpf_DTQ zRb~0>l!b{hV047oqRH8dgd{jbLnnA7Ebc=1o>jMu*b4{YmVLRR-%Wa`K^bDhP@%>(S8JC_)pgzP{2e?XbJ4TAI%zWJHnH( zEPjBk8LQicj-Yo(xA|2&4!hCUY*W)LB=!wT@9z6H#%W@qTM(dEH>Bye`jZ%lLHOzrdXX5~^MmncGM^mi0%^ z3R4D&nujH01oMg_|FlF>(9UPw%> zU5U$F%`1_=-%su;i;U13+gr|kWKYFF@UDg}Nm{DG)H3r+>hBTNj`4SY1ZDLe+PIJ1 zmyK6?A4)%?L*Ohd%U;{_q88PsFE!*!5-8sXQTD)lxFHi$4~en{WyVOwOW!jdpy78+ z^L4qaGPz9{suAGFu)X!syUdL1LMQ-2t-7{$Nv%cB`pPEw|0~t9Sef%qta5{r!{-S) zch0Ma!$D_z-1QYS)p~+7myOi^&!m~XGYpmN?&mHoAWoEibecGtBv#pA7PrgcMH)$i zTYkGd05P#IR#Q%jtz+N{Vqcujwuz>D+HRWGw?8os2ae(O&ZGH|U7AX;3+X&#v9x7+ ziMcA-kAI;(5UZpY;&NrA%19wwUNdj7|0*UA7Jp3j{n|fzOFiC$o9m3kiD>+bOSdWk zFwTt#J~Ass42Yj;>I5RdUn9^+2BiYdmv7BB2yg>|K?U{=3PZl`0iUP3yF?B+9AOt~ zadS8#3uUlyuTIS=gDX07696lfdR{-0qM5j9@NyYv>2~=D>{+<_YS{RQQ>rfT_^9B- z35kf8KDzJ;ec>uh`~dreOjv#dP2SQyGon#`F{8lW-C0ICl~>x<1<-sgt7K(^+{+K+ zi=`73ggW=ifxh$;_QIV+=(I>tBXB8Ej;+OW^8P1FpZxK|Qz7R?cr@U;Ba!i3cOx6C zYDGTfP97~4s7%Szf|B1{li~0Zk;_2HxY{B{Y6vjVA+YBtC57D)20Ij*MoY|$B2jV9 zTyg>kSfu&AJhLDI@Kpi$g~ps}?WBaU@MlQf<#I}Z?vn5EoH+_;9V65`Q{qA^)=p&e zmItj>nSUqC>6M=M3{5sOaGRvl&oY(|7<; zJnC$IsG+UnH@S>oJwC_<9T(I$m=|nr#$C<#-%}NrIt)@xSF^)WcIZ1!)zN@~IHu?4 za13SVgeR4iajf1TAzBH5N9IccLJ2cz5&gS)=^4v(xoJEj?BV;mx3pRokNs81<#76O z#2Pt_TrgNfm`)v+if`4V@r?pdN%ua^z8WRX4uwO{)67uJW?_2`m*34Th0+7KW;cz@ zkvJJS10WTcJ0Bzrmt~i90z0dcjZKR4=y-~$IKP)LIX$lc^= zC^1aRwHY(zfk=F+y!HtjyA4#_AdzM{;%s`As2KF)bq@YP5mis*L`?Be}?k& ziDhSkceeRo4l2Dh<>LaHn)v(ypk&L~33n*6|9&lU_7o7=T|R;2oL1B1Ebo1KqB$^J zqQM_2z`~$~7FW%EE*8#5SV6RpJ~~lyzAiHM)rBOU7u>5>(1~!Qx!?yohq=HV*|UL~ z#bB_%_G4>Ukv|UQ65pZCl-XjpmTa92V%4;l|J3mHyQ>4B)6ZaaS?7{gW8;l;BTa^4 z2N$7NnnK+Dmj~#_H9@tX(?Ybv<@d3jH3qz(<5fsskQdL$7cao*4=&_ zTD}zcWOsgTO9{Jr-0)ep0VlN-@4jNiE6|?;aDB~1F6dOZFX$Pc$ANCJ>aNFcf6`Vn zr>NqYB+Q9E|JA6Q8>*U!tT{ow`&OD(AzcpU)^(G%-)rNL0wW_}2jLE3zR9UIY#_}w z6AvWf#z$edd~?XVru$jP>`=!5twPcOrC@^pA|WJL(K=&Fm>7iBdq9JMff@>WeT2W; zCILO(M8p)Yjwe`O$Xl~m?X}|r=`W>qJ+4wABY9|(sxg6&LXQsaQ$xSv%+p4}ThlJB zQ^b&x@aR(b$_k;H=+TK>W9N|IVv@ISvp*0N>3kJn!34i*!CQZ!wGzzK&9=YwBMLcI%F3u0qp`vD|aJXpqt3lJ5QDt4(PXNA-%6@+7!wkU-2GF$o!LFBBG+2&F{$5?# z#*#GZaEc^w7!Pj#cuI#oU8(m<%+QJd%br(HJ{gn9g|X@ObCIRUdgT|M@H|fbUd(q? z2qkbpqHJy~+nnO_$ zqr(bgGHMhtm&}(oc83@J=z+R#?qE+&$8Dw@nbxu=42TfUw6fPJ?-$#at!JC7&vP6x zz;=VVdM&rrq5SkO2oL_RekX|wV}u<9rdO8?)t8BbL}w((`)(ic z-RJ1%-S+&0tOA+*!{Z|9U9tAiAlg_cK(f}!TOiByUH4-{pJ9YQjiQfqhVv#801+-o zHjEit3Qc%sHH(lykrRj=P9v{R3zoJL7RNW*G3||ZkX|9>7L}t9xXPcon=kTNuEf^< zStmMFDFNRvd9Q%j6yr8NK0f(<{+0f~ZC1-WT{u#h&x2;t`lyC5q>CHlp;qGBSD!|| z=H0twn-x(?I{SEXHslGUIA}6DEv*fFAfvFr@^YGtboY=G1=P(7YK;0+9Pp<9(4EZO z>N-~5T7s)t=2bBuVJjz~5Y4 zlI;G~gkMt{+`9u2O#CS(q}}LFLC3ru7_d5tjQkDuG^tIs4GwMxkpgOMOa}4CLj3Sa zU0CHp7>xNNadoOmsRP2XE_Zt)!gEEdAx#;hxu1!s3BL|RlPIcimLjX5h?_$jcWTdU z;*DnWepqe~1;SsYwBx4HL%}oe$fqoXlN%Lg>N9Y1yfOx~w7MU!z;w8ph?rWhD^n@+ z3(yGi5?};UBKVtRCRcqZn0Vb+0X@h}kuVsMt{&_1Jd?xV?*EWA*% zDr7Ao8Mu<&U3CTWkI1~l?sjyJ4u6}FS4w2wZ#m#`)4IfAWf?T(3i!;8k**-;Y{s!9 z3VOrMt#R|v&=SpRh9+6aPZ<{-)`ANTkvDbkXS3g5DDQs?&=~4tAU~FW@X=s|s|@{@ zh`=N8<0u>QbIR(&+G{7j_v@ZcVIX%~xA@p(E8tzyIN3lJ59pd^&zPIjil72AABWpK zCPoG`J^(YUe$R6ex~TP{PKw9*`z*N%OdIjW-&8WkUVE)GeT zEJw{H$el}VUaIBu_9W8K*dOf_M)&}oru5TkB_9QDh;`Ch`1LGRAN8iEeQxWb(oCp# zfe&0e#*Plx819QB68x}h2ytmH_&`c5UfvcrQ)MY5yg5yoD}jJw7%$eIbKull!ZtzP z(=@~j027l%MUzNzg&q{|b5jaREH(K4FBN(K*RDxW5+I9JGIc5)<;(Ugj9CvrxX6OgF z#`tQF%TW(g9Qv1`7Q>-R%4)y;rf-b8-T@&4B_#k>NHUT3A|_0(3Q^$_r3BiFX$Hf5 zb6^B|ZhS6efJ|zH46O$B2!l$&w*)OA=@C5T&Pyq?l}6<_(p-xV2wbDRyZ zEM4*CwAS;&2cAq?#i@h;J6-+GfbfaTsGt!bcJ59d)v~&IGc8xD>ul1S-hC62dvCPe zDKhA0Br>1DczT zVA#hT0!+hK+IRK%C$dqW!VP3Jmnut3VynXvv|7qmwT=9i+PU-t%Bxgp(RG$SNfK_a z>(bqSPE9zMI zhp@5x@+VLdfpM_m-9Ew&Ux!#&MnzX72}g3whaVwnAEAd}S#Sv20wJ+5h^Td@_ zb=0sT@0KO#>}fh0|9R|fF#7u)^8VaFQqDPD%WHh1<(S)eF_|~bZ$yrvC5wt9BLz@+ zHF&c9kz|9CfPgtVW2;q(jKQQ>WcdRYw^3@zVK(C<{#d3m5eW*43Bp`E!gno0h7@Ig zAF=Bi25Fk_cz#`dB}u|4Y|EOd@b5lAc|BJV$3X|@9h>UOtf|^^t-&(kuJ?f{Q1{F$YUISx^86STm0t2$Z1i- zzVGH0{G&ZK42euaS%fw%Y_Rf=U><#Xn1B9v0gTJ;E%_D!fTduAQZ}NtD^2@<6jc2M zn;aCa>j7(KrRAN|5~*i3X3*Yel8~lm=Yhjn87`a4wyYv`_r;w+QTe|mh=c|qAq$3c zcW=nsAY?t#LCC~h-jW(1l__?OhNw**;*mT_s;wm~+$1xVUd{^7gD=New)(wrv*I(6 z{^<1td`S{zswb}7q`9@7;Xb!3cDQtWuRZ%f^l!DssVE)IzlJQeliTD)fnls7%g8H` z3cFt8%ng?WT&Q^RNOm_T##si3g&HUilffjbP6f6k7)M&-G=e1C&E33p0Whp#PIH{( zNTH=AOZ8g19O(Vma~j0v-FP-pP1jNCYq5}P=QEM#Je>;<;MVx#v`C=uq=Q>htx&Z= zn)0*to>n)fxs6BDLavHhA?1+0+?WH>2<&|MQDMp4tgH$f^doC+nhVXN;4S*XkE}@Ps z9GCmu3JD86W<|;swsRo-LP-k`7mkz`K?SCYN@kOAfHNS`Nf}*1H)Gv3Yh}Wx1rHOA zy;C#`lYp0y(JffuaN>rh$FLH*MFipfIEOR^VMcdhFhaqSLsmnDhbqQXp<*1|8Yo!c zp-}+n42X>go){^j1iUj#is5*nMXL_|ThA!~UC^JTjtfmv6oanqKH=1i1wIB~Rs`aA zvV=Rr`~D_=OP;dtG&B%9st$oHf((n@`HFHAL!f>`QNT)97N{(T(g^*qq9TtefIVSH zjg66oFaqDieNasgYl7U1-YmIW#|=l4D=XI6+Ok$NMCMc0Jj0Zzv@tD~hIWIz-Jr>u zJU6{=<>I3f|2vPu@OEmyNau85hj0c|mDB=|r3)rkNOO*^zxZ1d%! zNRH$xd~Ntpjx7&cHvdfS6%E^)+p8=tZe{+-WI`DjrYt9MI!T%^HfsQ}#E4W5s%LqV zgv`Ajp=tYebTfN`2IAtSf!BA1D12iu7lOMqx}Xzpk4qds;bKs|&!?>q|KB)A@|Gd= zJOh@w;I$2DE`sN2ZYQ#^|BNW}U$+k8FXo`p(_@#i7I};ExyHA1p9l#6qlOf_t_!tc zka0;Bt78p3elR7TAt}kl(2X)-3EdX0^h^?*L!!=I;Mb!DV6@GQ*1!O6D~Ns>*1ZL- zfFx0Fb^c>(CERN$_5A3K>23h4VcJ~$o+fu9J3j3xQ zIi(K^GD)PD6yL{bF=R6bzEFTJ%2PALkmX>OcGm9XhL?K&cB}6>I98u(LY{HRxwvl z{NW%C+pB1%uzRR$fBcW4gr(h|&|CybDZG)ixqVl^Bz!0euL!dYm*al#-`n;VfFLJB z{ZbTbmbb+lTBExmdG-+z!opKG_Xc3E_XRca}$f*`c|yxMV#*; z0!wO?(`T-}BUu*QKf39W?1jM$!x(R0dos9mY6|F@saY7QDOfZBpD*^zbrma`qEyCd z{zNEIv>Rn1+g|RJABBuK{xVtR$M@0|An|3&PLHxNl!oOn*wN?i1xQZ};afbZ*_4e@ zw7ev;`;TOD)8fYZ4Y9|XHu_sI+n4h zxa6YFV`fR#GBY+oH)V)%(Z7(=){xmmW=>oDVcf19iP*WJN}QKgBFjh*Zk6Ze(c8^H`ku!`nr6*-Kotmf6dex^PWuT^|@ zM>5$W9-@0&Y=3P>n6Fal8&OXZ@FJ0D5j{xn>tbC!`*47&uV8r_KbipYk zmQbQ(YphwOwjJ;F8OlGwm9i!CuesVR`&-x>c*7(?mqBPFs$ru%+_w!O_ zdPaxnFsd54Ao!*zG}FPNd*tb(TI65N_xjivy`u9i$EMCH($?6(05fK7CMRD4cLrO| z8A7imdU~8`KN+z#9Y)If)2VkjYkVIpTOzVts zE(L*PKaOL)O!!=Sz6}O7X{=k7`T57WC*^kG=M&;Vyg+)aTFrFs>5KWl1a8Tx3rCWU zl^_dhgeegOKj8jZQ3>VsJ@ROB7b*#>?!{9aYbjBGfrd9Y2lbRoDkK3llwKA7Y8(#dT3hB^)zniEq#Q^~A zdUb8d=GaoGSwl#4xAB}+Tu<_ANSS#GU#k)tO*&b=iSuxjcP3q(*cJ-7AH^tJ-eYYO zUJjUmi|EC1L(GqX!zwVybOT$>K4hxF zuIe#1(iR~Mg%$1PP5%z>lbWYqjJ);^VS3>-jO%`BbW__>G9%O0*Zjx;bHS>K216e; zURH+_yYoN<*vIeGjODosL?^_+} zPZc#%82)^_S}bDOFE&J<+P>Eduu%T!;}*7;LUKXxm+fZN@nJ|3XZ+&dZ8!mtSO<$k zZv$)rdsTNt-f-<6J$c^PQccyAYk%jfvTZw0$5RLrRHe|qqtTo}XtmF}%1?6XHvBORy7~*&$DrXoX_aUz@w=3J-X!^O5zl#b_$4 z?;GhNHk1lP%AXk4)`x=9k&?z|=abML_5HpnDoWDP%pqe)KjZ6ss`wA4Y#CsU7Ye%U zCn{=SDM?z8rPAjsoPKvF+zwb;-HMX4uItXj`BGfHe33|7SP7+FI(OdlMvM{%EmID+XE>zml#m7F09ri zMg%`sIcIVuy$Va|*iGnJ3*uz`qL3UK4$r4bQT1s4xzN-BxuZ8% zHd|4a{5x(pxRE)TgU#RQD6l2|cz&cLW~2lc{~K?-GXf@E!EWD>Y#@OjZ_xtdNRP3T zrdEIRBl2!06?W|E1P&}s+eR|qZ-uqV@&>nl%UfH01NSvuLYbpdjIp_YhJ$jYth$R| zg4}1=>Q4L1lFElSzm0V-pZ-ouWT;84JE7fTvY!Vg-QJyr)i5uta7$eGDPuf2=0Cgy z+sWuMAbShQh#I%>QhT1EtPsZHjz@OjZQ_%E0K~3TLgtN zl3Oi~m18K9$OrN~*Czm#ZN>TtX;=jhn}3;XF?bm9Hc%9!GPI}J;Ah%w*7-1d^tY`P zcM2J0bW8!lh*WyRNpoQO>W0nrd1x%E4T?0XS8>u&<0Oiv4H~R){-fU?n7&V6sEc(r zThf*NGsA?62TENou3m}(rnu?uj{mjxTGskAsJvFGz<_`0H6oz*BddWaiYvhAqCwdL z9-x?+->$r=DJMw#97DK?HF4rDS(_o;l2&K)IqqBb!OYlPJv?k379|v+f{rhOF=IA_ zF8d~9XY;K4BXU+tb)Z6|9kMQ~>gAcG($hv=D>e7sv!-KEKB0ybAz;hq^fR%Ld>IC| ze{)$=n&o4SX4@IdeP%b^C*rDzi@78*%oZ65`e4t-w*(I zRIg>Wa^Q4hQB|R=y;^sJCC62%l!P|H0utz9zTaVm|J~M3aJ?y?K|TI^&Eu$WkAN#>{y@pUwCWCw9@I$> z)shH}PAc_IPprx(#w zrz$D)8N+3Jl@;YSdCvVcwa}svecZicyYEv&grkWjZp~?JZQ;uAE$uH!(fZ2q2E+gu z1M><_w4$a!(ob&1Nu1h7}Yel8b>I0FMZ;0#Zj~@-c{69836CW)cRJQVEwnKbo;xDSsJ!Owgf*+%PrG<4OmKS*#`se1Bht;wkZOCsE z>0r5sicFrxpKZ?Gm}e>fp879XsW^}z23~*Irmh6IkmRQb^iZUg0DN0j>CEV$i zFL5XTPYE%0Ob%c# zX~wp1H56NgHb2>CWQ}DkRVVhH%E-x9t9#X}rPs+0&==oD&5R_uX0cS2+0j7462~10 zle7~*Ng;nu!Ko=_$4`T2+9)zF*Ov2K<1U_X*@LGLTS{3Ld?vFeD``!c93*B1ifTqJ zEB2^-5fyPhLU&%tb(ctRyAlmRw*zmnrF2`tV^0^JRwK_u+d{!Ws5}!GqdOjljaXGu z2=kyl3cbssEuNhuhFKyZ>dmf+r-6z^p@=~RrkVBUuUkqoON%zW4({-MQraoM1kB+u zq>$5YmJGQW0<8}zls<Gy>15Nr?iH4*=aQQB69Zsq{Y%?2f@Hc+? zNNYAjRT?Lp^~h*|JCjV1dyzt-l(nFyuGvy9lLXj}X#|Y3`+W4D6v9V)))!9<6U#f; z77hXwvwMbva2yj7NpCJ^%V@rERMk472~v-7^0nYK6!K~zHYl;vKe;J1h5tzEKu(h8 zi3fb?iA9K}fi*H?IgxT_lX(+w@y+FkDuiltv|-6b5>5%(GC{C0rbmfnhj|m$|210U zDFs=jcJiXa;)Wv{5z=X9g$%5oDy3;mx3%SNl8L31)~m!jC`u_PuRUfFs;Hzo#!FpZ zq$}4b8e3YWq@^mznrp3OTi3_*uo(!}67&k3x%@#P!u-z8=G52JM?>NG#T)l{ghjTI zKEz77TNO~`E@hd%R)6L>bY~uy%WSg0=KJHyfcRX)eD5GIGV9-Z9s%e@rDSz%?l6uR zmQb03D(;*U+jhR=?FVWvUm5p~B@RUxYWNY&N0)4=?=18WpOq|T+#1vYMd*p7aSw|F?Voa*0aHG1Eu}D(wX-w?6>t z5l*1H6B8|EI6MqPGB;MWGhQZ;pDVqAbva|FOi#HE1@PSO=ngQ8NzwG@Sb;6o3_}f# zh@nTU5xk8UDi_y2+o;Nt!HyDi&3N`>LSMJF(jf$Q?N3DfzaLdq(Iu$^stuPUmGVX8 zEEG$~G8#pJATN_vgf;DbbL$p}oEGR0_pE;-yB}L0&N>`16bu~|k|$#&S~beHb$+t% z8#CO|Xnl``*tsm7Bn%w#vWc-vBG#pHoj2vvb$H}&V~e1{>ljcm0Ep<6X869P)=$?e zlQ7Y0tU3qdx5uY(lTg#y$gRVVV~`l3)*HdO+}BiXY01nQ$p8)-9-L`#;HxvwD@_z) z80sp!8nSI_O6pjba^X>uV8Bui#a!DTPS}~bWavv(|GgYf!02sInF7$<@Gv&*LllV_ zIjL|Pd-Bb>@Mb^o$NjI$*^ni3eZIy?R(x2PCh#fiwJrgN-7pP4N zsMsk?8^uB2YmxP$sw$nvU$qHyXsOvSsDVvGCRm&Pq{+5Hnb5(Sm|PUV`qdnaA_fR0k9cE3O2XQ~OkiU~r7WWb-v_ zl-6h3OB~6%+{c;EwKGBLF$;%RjGZzgQo2I3S}V!Wm{bGm;LwH~De6S*C6nV1ydx6V zxl9@zBRg`~r0i%r$ zTbpXT5k0IN^q`goH|)_kd(Tabu3hhHZ47j4a}+t@3T>izp?gAcP@V%P!&s!0O?r=p zirq(z2*0jTr8>v*M3cHi>gRqAQglEfo~ylqM`p(yjI5C|19l!*XNUog+}s8xwdIwW z&0`+}m{6GQq{S|PMa8dWAfmzXi-W08qs?!3^b*R>3B`oLXib(Rbs|&4k;$d);TY>W zxhRaM=EU8ar5`2P=y117hqu}st|S^?oN(I>+}9U@u-&^I*v-mFDPwcYz&}R#C^Xb= zK6}nuJ(ylSV_`Ukde@q83Wj~!30~pY)J_0YeC#TXxGdoZ97RBcjb1J@S0nHT+e8k& zd6*h1K3bKAKS(H+yFr^N7J0oER#SeqeqhVF)#NgzX!TZoECxDd$IkepKuMn4MQL`U z>2Rf~o2fZ@&ddF9NqDuU@NMl!cl;YP6G%RFK_3TZJP08@YT8T4#YPET1Mtp zI`9yT1@H%2KcXpcSe0m(s^Q8A#c>gkUsaAQlSIJ8k(k?MIoQxg= z#v(GcKN(pkDc&%}QlQYP56TWpl!?*=c5qid=DeU-YFZ%n$Osq?`~#7JEjxiUMsO%? zp&rFe|4r-jC-1zEyC7sJmj&&JqFg*`%bt7GzfFAsqucpY)dA;*OVAH^A2FC#f>6W5 zcrr<17Es`m!hZ1ryHZ5aQab5kyJ=;>jpv|YIi==iEn5yFV+6^f>NeXsdJ$pLEe(rH zm@;qws{y-s4u4*&IXp`rZurely*{qK)jyiwOlBZrTeeM|2h33)&Y#5Q?l78evu56- z>&PsPzS}f&%zV(fo!xrug#3O^EyYp!_g9smt#c4I?5xqmVT2jWv|#{MxTf6TZO!4{ zs5Amv4ZTJ!1yf8&lLs%uP>G4Ce;dbj^}qGp0x+iRsi2#R5)*In5S?ArQl)t>elT{B zin`{V;x)Y5 zr|IkgZ1UU)dFpbZ>KW78$fYT7MuFfmEI~y|OX^aVNK?&BSu7dhGyUqx@l6{{Y#lASW?P*WHu0!!p};Jjeo4mz0i-bT#MDWvV5)>H!EV1 zhyyCd``kRE#G#4H##GU*5ExKY0t4*0AOpbKAAEm&izPHlADW^qqo`196_n={HY-w$ zGqp3pxa@WKbG!`AXf`0{`)D`Qq%tt>e;k5YQMgeOd`y08Bh~SWMnp&vP#Hi0req}q z!Y1c6uFMpfO$|O9Q@w}zgy6BoiGQ1V2Z&#fB&q{i4O3y!afQZIc}11?bLg_O8;o2% zhSWgSW2V^?021d$5b|PqB$U|L0$L0zJPN=hJUD?A(N62;a2;B_seB`#QJiUb=zH9_ zir{4l1&=0fH~|1SpM~D%7cmerix*)cY;BFSLX$SSpCjE!#}RXtCAgqh9P(ZtVp*W| zDUMe%lssi>1RhQyH%YB?Hy*Q$P=S9}2Z%QA5O=J{(lVKmmLdPx}-C?u_i80_nU}Ae3AGHbK60LU1*gt@=WnM^& zy-c(JU~l26^5Sql-GtB*5BW-7kxJ7HCwqcHaALY;48^27g|(_4su;Qx$LR&D^OYjUIsJ zRN124!3#h)=4kF1V&E$c`TK4^XUi%aNdLQAP12f+$onOON8GW|d+X(Yi2(q49f?E^ z9spPcA9!5@0F9@88K#L#1#HDX4AuQyB0ZXCk`#TTd$nY!3g7-H4E9{{pU!zj)jx1dOF}9UE*9}_GbC_zcJ1Z(n})%# z)7^e8QBk#f*XAkblZgJTD83szLFrM?=DOUhBw$?`Qkwxr9waiQ{b!wL0HR$WSvkD@ z&^-XUa{xeSlv!jn7kEr)H^RWg!TP7@H>j+u3bqVm=<&4&fA9A9_bp>(;Fd{!N?43L zPgv>NMPtxKKpE3YT6i&2z*w3(hQkv{DN=0c9=Uf#F{G}(^|Q-X%jE@BWAm`9c{C#s0BA0~#Oy}tni8cG;rPxg*kHaYtpx@ylkiZ50? z9{iP}9d%UsX8K!I4i21hLwESy=B4T)fW3?Z_`og5Mb+aYTdK;a1ONblN$Bgue>XG> z<+tH}+R^HWk%pWinr!q8gXvZrI#$qnujEV*c`-vr zfQ7|j&d45>3^n+dh(3Z*{1vGxBP54O%S-QzG5y_WKmc+YIFK6;fiqVM95O&qMSgu! zVot*ifb2ON>h7sc!lQ|Y0*hQCTp6wg_%tPr4%C^NN~jYNG8>kiy}_E8!J^Svs=`9b z)Q}=vK0;HeLv((zWNlkRNWk~{CNG7cqC-zi|C=;gAa%t@{@7;QQa#@yADueXBbE~P z1S5Q^sb1=LD}5tL{9qCue2T(@B98|RmtulZnT34ZDO%`17yAarVRxmd0(r;YHd^t2 z0mDQgL`q0XI2FuVITefsmf&;;4I1iS=E8hpZnwQ}isc|i+-&ia6>&vt)wv2T`x$k7 zkT=ObX7RO9xWE6aN3Ebwe&!A!-g(}G^S`prE%*43)i&>SP;dzHrwgl9A?mE8=NF{s zM>XpS7iB&a@OcM#tR_oQ18|sjnuUhYvr90f>PD3jQ>4m(#0*K1 z+E7llrmKZrp_qrNX!d%C(rqrrqo!UfrJmO{eI;#LHzqM6pMCKzb`%$fJUI#xc~K)C zc#&Z-G#q7$o#;2uEM$z1tH|=@{wmsw{@G#k0t#%e7Cj^|_99f#v%yb}jMLl^R7`cCnQN@(;W+2qA`LW^9DtqBXz zO4U-g5c(x#@OS;60e-U*4gJi-+f5>1ZYkmZ8T%Ye^jxS8<$9v!tY_(uDB;x& z6|^`8z;Z75q)XEf0@aikMONU1>(IMsQ?;y%Kryi^QOr(n9P)8U-oPvKY0Ej6^FC%h z?wuqu&(`Ov;9JeJ!7t-z)$F!>@dAN^*IYmL|Crz1gI;T@*G0A=zfi+kQpB{*&X?>e zc9yMFcBt2{pZ+oqx^}k@{Cbn`dh)%i{QEv<>NB|;06%&qMps!%i)FAc>1jq@>F}YE8!1X=qQ{J# zI|`*{i@-y6W8i<|_K4F9DhdcFv(u@r?9Sh+pORLV&qc+S|tFAJi_ z)h?(2ySRH{B2#WaFl zrv3pd^6(L*q&sDWrmX5O`BbHPVZCP^`h5*Tzpg*~3_`J9eLo4T3<~y@!?k8uGO@pC zc=M-aiaMyv%#(2m`-=W)!8$RqJ*KwB{#~kz7BfsE^|>gC7?aULkpo|{jFja@pt=+} zdQVmFvLVKrM^hsGgu4+SY8A`U7P>;@(1gQ_v;U*_PUO0=H8MHK&8 z_<4QlPab)~y0|7{5E1OrTG6TQ7mB!FmekHKd)l`YQGtlwJ7DGTB^o7v=uT|HWV-GR zqMe>|%6m*Gn7GQ~Kq@Ls#L}VY0e3@4!C~5EzrR+4cP8F=5MQ0WVJ>IcDZ1)ea_(St z%`-17#J{>(U8S!p2ZKUT8J329lZ1e~L}T5ho+`;wgsf`*YKxwkDq$u_m0KSN713>% z5DT^E^tf->Ikyx)it@DdvfC zRT;K6R$>>Jwjl;U)yScPRg59WJbv>Jb9K`1&v2;IrzI+}qMRW z&uoKo|O-SPMf@46)Q zO`rc(amm6@5!sL!!y8=`3MP`s(I?M_mpJ?mYmi3>XRoH#*&BxNg@3MZi3)2)R7g_! z0nQ&USe!J@A$s|&KR2E2)?lTvMR1NVQM4|`P*c8}oMlQJkm1!LBO^h}1po>l40`;~j2K2>GH&Z!s9Dvbf{h>Dq@+mt7dLq>PgA8yIRcg}-hws*>JCFK|AI5F-Gi&1rBWoXK8w*zg z*QfMbzOWMQVMx6&U~2H2tN3OSfEz7bab( zphyiYMw4FYB(%%$YtWN1s-~R)%ZlWU=Q5zq5#rF0H6_H%i~*Twomh_1LMS*H5hW+v zs243xAG^sN+}fK{!gw5Lx+iKJ^Kp1>Y^R=|jHUk>?goHE0#(%ly2mkbu)IYifuuG7 zdr=)WJ`({>8(-IIdVkK^wsD(pDIE6(DOs?tR$hF-&vJUH`}LDyqrHDG7UJJ%Y8;sE zV!r3+d~EMkDIfA**X z!;DH)tBtNgSn2{%zBLp+i(^nq+HAzw#@I!lotnQ;0FBb)59L|w? z?c}AZbu1cHCnCPyh|v{UH@(Z{8=3xlq0;a)P+cUmv1c$2Ux@xC$Tdy?bO9w=zKR>D zpqb{^RV5aDtSCHt&7F&8E;vexs7`W{0XO`0N26Rd;?IeILwwpsLA(n^o20rZfXCSl0mnl zm;U&hY?oCu6`w`BXgiU*2;?WkqME{`)XEb13mZG;yeeg-`Z-r*tLF&?p)7I6 zgfx)fVGBcOYeB&|6v2osU%n;NumyOzoMkLmg&Wv2Aw*Ynj^9Bb?m(uG(xDW#pw zVSR0HE^pnXsqfh1&*#v?apwa}f^ zlI^Qo?-$c8tl2ZiUr7^+ltgj#TOfdOmn_~_)uycrk^>!NtLHZ2(;+BHyHp<@6?^&* zo|w0>E=up5>TKKbvMs!(G-;V^eZPYaNpB|BpA^eIFvY|_L8n3>h3m5Jot9PVwslkn zTSZ)i)_R|g;DNIV_O#sKE|;#EiI`_5^CFP@Kf|2>aLl7kprHk-8uta1$9IEW!7S=y z$8hLqn&Qj(!cujKeJfLiW(eoN+NTDo0QQU8_R1`*o#}uOF<0RhGxKx295J!U-O13! zc7Yorv9`gR8FW&~L!?sc%p|9U%RJn;<2;Oux1WC*iFiN$0kJ1G0TR{{_dcKkr*A98bQVY$zD1#O0ebz0-Gd z6E!kfVKMRem)O5IW%=$KWs(f|ZYCq8%<}};DbNZm1ac9^mV!jo!|0A_rk5QeuK)py#NU28q$#>{^igQw;-bt!-87O)BfpCQdp@|#>;9`dr1x!3 zHIiQ4UlezFhU2g>Fn2n;6ADM4xU#En=*0SPk}6SG%z?&td^Nx?4#4-OzAbJFlt zB3UwkKuV#|2A#TyYXq)wxu%(x>rYB*m6e-ix#Ups_1_UkrOoeuCk|>jghKtxa_oHpp7Y?0%%It4O%Ez_ zw?|6*y!xf)hVKlf5e)=lo2L)Xe4DLlX&;1ad_MDT+3M+FBoOFF*Zr<#d|h5Y&?)xT z(suRH71F6~&QP!HaZrhXWillKhA}aq&^FO)=1AC6CKX+DLTab)<7FqxtileGCX)=y zx6cTQ06M)JT{7SjL^&-B6Z&Bn2K!qN*s}%mCOJQIGxq}hmyc7_JZSKtRX38$IC~w$wwb0y0gW}K zQpG{Yt2oKGN!;QqWQi^!RLw1PkEHO5LFi%IdbUs{rA)XOk51hrH-p#B+<9xx{E>55 zfCjfkjGUd!!g7tuliaMadaBI;thUJ)dirO$ODLm$hWZta_r4={Rd5Fj-?w(;vnMtw z2Ab5cxmbnCvW*CJ$oG_u@)3u#^5 z1>W_m04SQOnF24|0C1cO2AEuB{cDo|@%gP>#!J>8k6o&&mI}SZHnjE+k&zJ>7o(lJL)P7hs&>MA7;^#C=MTEnj*PUu$I&2``4; z$YLo$fL7a}Lx9#_pIDhYhdI1L^{aAXDly30r3?%@o}fHpVNkmyOsLMPs-eaH$8W7T zZvSPS(pW`+!=PI1iKz)3kwt3UB){Bdl{#2ejao%6F_9dYR(%_^zE=Q*lG+759Jp8KMsY2wBd`B&9AZK?K? zqyT{A@Li7DYfG{jtw6hEq34OKdEAomiFpU{l(Nz?_ZMObz=E({9y-csg565TMDd_L z!}p5u*gEb;xsyD%Hrc*SLM0Md3f{C5CrO<+-Jfd!dyY3&I4a*gxwA+zJ;CC9_&pwk=)z)8YO z6-nOVO9(}SON6s)GY^Ys^@ca)60CY>D-NZyN3nQ#n@5&WMNqRE9&gfFkG?e!9VU;% z3>QstTS=N(3F2&7sQ;PLC##bLS9^5n{6p^l0#w+}=EVc}p2DKoRKHLjW#`AxRJ1J~p|#K~G*29%&* zA|5A;>-~kFa&<{-ZETXtD#8pM**KaWKC2>Y2z()RrRp$kW7QhvU0&KzI%;bl4nN(>lY zDMq5V>-+JD`Xd323?V(_pQqIF(-UXPh)60kmT0A6a zP;!N-5CaaHqEowP?(jyOc1Ex$Asca{PN={`WbWX>93x?_Cn)eyh?|RMo>4uPdp(r`x3U zQCHFDIRHw81RohI!I!#*qa>IV`6Wtwwcmt^)=vF(PSiS3xF6k1NOMlz%Eti)>NWi9V($nPi3E46N}%@K)zjjb;L8fSaS=l*2az}o7;mLUWYc)6U1DeiG} zc@zrF+$zN#6oSh(M^_^sqx#5(&U1$Dieu}6vi*hJ!~~%6Y%iN>!d9+Py&Gbxwc}uF`10^32%uvi;-5QB_yC^eRW8%+u29`JM9VcKcxF zb3R)wdpugzdWc$ifVBAcr=o$tjk`OIr3WbXX^>5VBp1vZ0u52Ln^_fEJ#JS&AJPcN zir76&b z%nla@HjM0Mi)&+VS<|dh?P!UZ!h)3pHcQ(B#Z`7LWi_Vm6Nc~SuaEMBkCz)c>lBg< z(X!f+K}*p#_%;r%<%zzHf6p3=PDLexWIP z)n$?=!W`YZhl5P??TOAI`e*IreEEm5UIOrOa-byLSm$2c%%W?wqGuIvLZ2oOdsT`Y z5CGYq5#+jwP|ZM-C^<5RyjOa3AK7Tf5fa7xGfbG5RJ+Hbd50;7BkX$jc1Qm66n_s$ z+5bklYAP{7@WtC4U5Qhg4g%SCj4D+j`&EDo5G)E4kn&*l2lgWhBPs zUgSi~2q|gAL`hW0ivm>~YJ_w}vN!=N?zkbAcLv?U9DbNIBGTxHh2`Lxo9^~pci(Ro z!*#_e7%!t`ZZrV2V=Fu&EDGU2tfgiqHufcJ>oh{E)mr(5y7{qJ8~#d5J~((wM1Z!g zi>oXN!pMir%8(o0(AJoOsKx!J{{N-X09UR76%wZO6gW02AP1|4XF?e5rKKhyHY>Nm7PPZkzW>m6)ZnwRg z6)0r(vQLmFoBl!Lveo=CRTpV??pkV%k{=i1koVj!O3AR$$IaD>zTa|3wF(OY#{vje zD~7P%9NDQ9g%}t+qweC+b09GTaPy-n=%!YY>k7&*(K9;DF{tO3My6MA{hj|5q6+9u ztM8~7mZ|cNP_1=WDum6m6TDBOfu8CD>xw!JNsc798+6~mtB0Qy_|8d`(wKukW9~S6 z$PEfH;r2(1wJ`{}M;@p7ps)z!TIg1Syrs0C8B{;=h8rxX-$RVQ)~i<(hNP4vFeSz|-JEd5n#Q29*>1oU+3r>C z7H~-9TKJU0nN5v;RaJy(S!KcMnlCMJgcpbSadA&iG{#Sb00Erx!tYHNKAh}+Obv-%JEak=bN<> zIr2($9`(XrVA&!&o|=9fI!bitv~nWCH^lkmq`z`-FN@DP)LDa-JJ2_GuN1<#k# ztEtqs*oGtAxo1^O{^8opSyc!SdLyasBX=;F7VC-qX0ByX7f;$Dtdqy$JTF`PcUw4# zWAQ9oSh9Q)l@~98rR9$RV%u(Wc-Oz^O1vFnQ{get*dr7UcRWaZlBX;c5-5|@?Nby& zl10W>!WfPerlubZt8DNM1@D(}kj$W=>)g%Bt8*@RM1S(aS&$7&bzJ!%@EY7E+&3lz z40q@-HODP;hr3j<2s?-l6rO}`E-df!ra0VijNqe8N9{)eL#gE|MnWsGiNEzv8AZsB)n7HM*x!Kdo; z?W7+OQ4VvA6ur5?*&QP69)*}g0xgHIX@SZI0+Q971ETag4i2>x(_du z&tqc6Ud(6O1bt1{Q3uV!-%wL^YYMU0RC|_po@H3{q|s3-?4p}WBnhsVeFOj!O;%ea zl%)fImS8i`C(m_8v(${FO^$1yQMxlHC0V*Dl z1lj#sFaFKu;j-E*O+D%l<#Whod$J#K#Ceh#|5uW+=RB%jx zQUdmTrmT+jc_rxjs`{^^P2v8}^}LB)47!k_lCt49g3u1(em({pD*@pIY|Ylcoc{U& z6tl28@ST`;6jm{6B4tKv&(!U2t3*LxXfgty4*v%UB_yqHR0v~)+pJtoXrL0+g@{#c z5`x)?;Re?+v}NRyC=6t5KArd+1^u`}j*;kmHcZyTU6j>`8TI7c3M0{|FO?U^)x6%l z?~D_}v-S9H^KH#eeqIzRSQje_6X{+x6L3f)%j1;Qza7LT<&t7-{mt?YyRLpVK%&_z z(mf2Ax$%~gMf;VHlr$sO*D?*YS233?ZySh+{bM71XPu-1?m=>h(=X&A29q+moI_Oj zwkhK|oC1hxp$M9HiAp;L@7+w*E9beGEOnzEF8?*$9RRrHs(HbSy9cg@JNkX@j{STr zxNs*gYrS41z`xKdkgA=Eu!0)(7!J$bqH`@vmK7oHCpvledQ@?27u9kVw=9-f&jS{baq%Vls(QIN*5U zj4M&)oeHWMjedYs>xWRaAymKP(s3Voix=X-8BWxOt?i?vhO}uYm(oCCzS6{YT*6e? z&~fp+&&==M`^fcWeGGuSjt;R%CF?RFAOt98i~A_Y7dhKz&et2H7k5`!1Z0HcxNGUA9NlC6F9?WT1#pPUoZkYL30}uS{{Zp-*5{ z@?uSpJq9@R-qz<^scdH_A)IH#X<-fYTEv_`t&sFQr#B<>hN**KJF^p+&g5`cnQ?&v zVwyB2fwUGaK3kQ&LU8I(n(}ml6#iRvdlZ3cY(}!2*e#GMx%f;V3wf>F)=47m;>Azd z5oyarWIZBc$5)NgCC!`lefRo|^Lq_-62jT3|LGtArDL%s+!v|nw%!SDT;`=(?HJJ`&=yRNi2}w~P zvmG!Tl(ZB(kbQ5eG_p5T|Ma3U;;CF(Ku>oslY*hcQNi+YW*6nLTzs_q8#ps?C{F!@ zxldh;MfHmGa-cV?SJ2YD5x{dcMXGfmGd6Bx97iO(V%3r@wiJ%XU|5bbYQ1G~m?}NF zt(vxfn8bzn?o-cGx5lTOUa%kk2VcX`drxaz0iTZ1ZI|am_m$622xgq>OhH#?njdcM zonBrafKM{~?NB23d3AzSvrgpb5&7-ve^i}DVz2itv?R$q54ug%-Gt3|1+>q9+l zF1X9?tvc(MP90{_ZnJkF0s zq0f;e4~8?88H=lJtT|h`)0QPFc9y4)Ig5k;I_NIgC?-o8-a%n*e5}urmJj^mK^I;< zk6XK&ZnkcIUsIQfh9aHWMg8o`uEhj2$EsDFJ&4;rbccTNcTxuRJl4P8Ek#HWZxB37 zsq-$IGeb=*sK#(dLCsQCPX`?y9TbrPZIjv}Z-C1BC@XL9zk!r^v-%3BAL44j2 zx|DiuxB;gpd@3z#dUtV4<+irc%XOS7l#i4xYNT0dTj)M7hk<~kiHay2JSfY9D!`32 zC&HTwN0RXqrqQ7Dh778(GE-xVBO$ZxpYaF`qS*5X3u5B(^Y_b+c+l&=R|LewEMAGq zn7EIQ$T?iUk#$VZ@+m2_qY}@Ker+OH^0ef(K)OjJx{evfe4>*~gj=yLJ0bwQ>PsmH z@%^wymDM3Oudw_+GdVZQ^izL5RhPlFj*o13O^BwQx1ftKN7T8e`!r!@y8*|rPB*`S zoolElku$JqrR(r!Jj?44`-m_50J{BQkP^;>f{V2W(>N9@-p2zNW$I60LiI`~FcGI* zeR~R3a5)>t;8KI3!fwW^cP9sezj=csC-Qz4yEV;YZ0I}L`%RS{UE1PIPqukwa}d$f zNWkueV2c2Nq#z#Z=-)mEAO%h_ULb!=8WTSsem}`WktG%6I}jU(AURlI5J(3g&E;du z%P)>W_wvs1Ma?u;kjo4d2GF#8;qj3qVIx&*u;42_#;InlU`6PA1y;_8ihhBy2V)ztIKVMadF1ZHibX;%WecZR~NtHAzOlAEr-C*9*ulSmKs+F z)+YY+Ui}t3F>IkY}1|Gs~I^5`UPSiL8uOB|ip4*~g-zp+h4hyoyF*L*}mKx|n;#>Ch|oGmgS03$_M36jR` z`7jo6h*QxwqT zxhB>Ru}!O5lO#+$p4c21rehYPs&*r=b_C&3Lc!1m;8lN)bVoM;S-gq0S0ou-!SKUh zESvxgrX0=_kEy@>{*T%io#bQ9BP2nALC36ID;XpFR{}S>t{&xEIb9w(uXsnlBzX-= zoEw7w;=q1m*J_!4sj)0oWWN79asTfA!n*1eJTSxn!4uDp_T@xrI=z5U3tCKz#hl_J zZZ$tMY!5Po*FHP}jYA0GeVR@_Qx>SvN_u?gUndai`}(B5BV(dQ)5!4<(OTh1VWt~E zekhTDsfKuX$Z7h^zbFpSl~E%XnR3$rR=2?*^<4z}OjO-O@>s-a3sCEFh`undaO4BX zC>U;eEeVTL4*6I!cIy{|L&}W2xf-@Ln<2{4veo^vNrQQF{d81{GL&t}Hk6fMSL5K= zG_W>l@u{q=1Fg{z%h2U1=+uz|!XKqRs1f;bbU4y{=V{5w6oKo6Lg3oA`qI)Hgkh>g z)E8o4Nc#Hg#?|Rp)s9r*lrh_F_1#Q_ zs8q0~@(9pQREDgYfJ1?Y7M+&L#P4JH60tR2evQpS2i<`l(t>3PFm`G|VF06Sazi71 zAchjDKn0JT4=ZteBhI(AXfr`nh%tcP;Zpi2Dv1(z-)Qr=rT!c)6i@J34}`~PJxb0g zpXiK7)86iEv|g9(BT^&rLHohWuEsjGlSTAD#-KC7^av^P57R)INNzh=dp*j|4(!)4 zC8g+<$&m>q+q?#we7g_48wG{6S+6+=$i@>bZ>Dy4lkLV+5#T8}8ZMw5I)Ee<-C$tH zer6%Br@9?XP*S9{o?=*DO~4mtGxJb|S}>;L&DMK%Fz$#py~4Q<+c_PouB|l$ps)P2 zcHcSs`g~tKnJR4}^#QDANQAqyUiMn>SKy8zYl_b}xq&jhw$cv=R^yjkE>AXUTEOuy zV{T0#wKqA6cSc+>26h@I00Pk?V=TiQ1P37Q8Dab^fE);HeJ90u4mft2QlFP=_&lJ@ z%Z&22l+p4A$)Bdls+3Wi0r;u^_`^#;K*qY+6xH?GquuhXoAa+SlV4e~8dlLYlyudt zFK}j8T%!E366`PRcPjwFH`+f5Rv(Pm&p=C%xDFF=8dJ0umR?d4Fd$$+F%XbrnK-lJ zWXr=rZd!~`LevJ!qxjWm`U`lKdb4EZ$2>M6bX69&80k@bbBgB?AY!D$Bdg#(b0}gY zQ|ePs*hr~W`PI54RU3hVHp^VzBY)y-ooD(lxAp^o&ndzVx2Lu|EI9-(Vm1Rvq73YV zdV+(I8R7&zS+n~?fhllc<|2)O{<_BYEV}Cb0XbwA)q738MAobf+-OGAl3bRh#-eP9 zh|96m^sJmCj`m3*adi2vY>`-?k>gDFu0h9zD~qf3(r-s07%}pd;T0k=ns3QqTq0F< zc8wr{PHIFRv?;67WJos3$82=jKC>V$A6B}H*b$2$UqYm$w#yJfrgBX8@SclglRO$#T6)OjQ63GB%)a}${dm$F1Y@r| z*R5Z3Q!WAk7}6&ZK3*K~*i4-p>0eX9RIy=DCS|00^zSJMA2+aBV2`lY(`8*U;ilu~ z=8{aL*Ueb*+7@6f$Fmp$B;^h=0Kig)O|_8%XnVtx;*S+1Mo=YI&4+1?hMj|mWJSzr zV>S!hjw$sM!!W2bPLfb@%C)QxOmAxcclh)Jfce9h<_!6==dmN#Q99paPJb7P&-4Bn5kpdTOq@bZ;|wetVNS z90VItlRioDSl-`(o!}1qwX-q#0w?anX)wWPW6E&unT<48%sx1)039}#gH4b8&{2p? zAASs%JXak8z6pOIVjdIec%KJfrk`rfg*B*!thJBgG&H#37ycD~Z=cPrpR2rnxzq$G z>9EGQ@Q{8B%*31kgL&R!?Cb&vV+9w$b|%sc*U{<_Sy{nD&c6J$jaBi8oty$@I)V+a z!$2_o{`eA#X>RM$*NQnQg#V(!p!U6?tLCwgMu#(ugX8OINmQhPi0eh3{VI&)V2tSO zMOs_ew_|A7^wvszx9Dti+812N6bO)Jmb-LJ>w*ZkC}OT;0PlOE_SS^LcwiQ{+Mg5T zKnkgt#O2VN!uM1&G6+fNR=x%F-xo4nC0p9f6^owKAF0ZSq8xY$gzYSKq;5Px4aUa{Zmd zi(~?Bf^UuT`%hyNAIc09t~$#s*tfnKYN+HXUJkSO;$4C@>}&nu+Rgu`{P_s)$p7;J z{%66^9{>TAQs^NjSw0eOXf0HP0Wr*+q)?HO!Ja?Od$l`~)dk z?5Dp~mEnG6I=d8GQ5l}w08tyxnpEkETYJZhhY*7!H&Q~_OTyI76Ur0 z!=Atd77Xi!$mvc6R>oWDoEwRgTc^Vp_sj`z0(DKO)z(KK>2&u>-xOdV4vuk~WBoo# z7o;UF=@fQ>e$Z>#>##NJAb+cf zNDF&b+>eR()By0V2i7HW+ghcub;BfF_^?b~?`qoe{YMl=J$hrw2_!PCENs&(r=&ZG$; zMF)hLuBM|6{*J)e!gq^q>k+0Py3ELEtWZqRzcy^svuN;z zZ#ymEL?wLFDAVU=048~T)jv#X`yd(KEX=^Hh&6Hi*kt;1tyEHE5CetmR?1p~UF{Vz zqtX!(e{iHI>q!m__LKd2=7$@2F8YTHUDkYxZgiy4y*h#vZr*!-!lD>AM})!y1?$ja z^vRV{Yi94_O1*JseT^LD?_{qz8l$oZS@>i5w3R+$)n7ox3!H_JEBaCYK8y*7+T-d( z(6CrGU4w(f3V&-$=(^utI`eyKaKsnGorG*Bu!YO5FKALvGFx; zSXkBH^ZZq_K&Yj z@13t!q*2wP_A@mt-^JochL4_j&9B``c zsjME5K0}M(zn7#F&>tL}F4G?fKvg<)=$lFv zV6dKLT^uY|*T_tn7*9QdQ3SUx1$hU4upw#p-7pCBP+2iEP`|Kw4^VWBwrSey=f#Rn zSJ!Dj!04N&6~iLI&sRyO<=LY$Yop7kCtukI>k;PJgtAF%y<&`pRw=(rHryYM!fqlp z{MT@o0H6fe_+l4ffT7_6o@f>*1WhFn^Om-N?r%-W8h9Q}6j~|kYh1L2*v~O_dqk8? zRwCI;**-fx)_kOJV&clpdGBzv;i#p4Bnf+NZx0Y+@{Dp`LZPsUrL6%V#sr3&N_ezV zCXZ2#E%DPLZwKqI%o$i9YLkkPk@?jBXPrD)x5XH>} zCT!;WEQ6Nsp9Y0%Oz_au%#?Fu3tRL}RwrDys0N;tM+^@*#MGR8V2$_{SO&_CkW722U z!`ee{O+Qa&lBCXbv84X?P6gebX?D}r^=NS89acF1=C7|ktGDY80lJ7I_jLT&uM;V8 zv#loT@w#`;kmS=He3|8S?7t8pO4mCOP1O^asWFoD$z>0qa3t4Tr>1XDgO!1}k#5-( z4YjPuF3vXHb)BzY(}v*07>`@8keN(Lp|0&H3urH#nx)qhA@OQC(&J7UN?8)MdX$u+ ztBVotmZ=a+`<9D*)bo3&H5?dRpV_+tk3@(Ag;KXqxA{cdbUqRdyE={?igme6QiJ zzTUr2LZ2i*?!5eT`AN2SFrF!OMIg)~rUl<(!zAG^C2YxFSlxBd!!3`v?XgZ$H6B2E zY35){ybcfkPxajQFB?8$O$b8C2WYgos@O(4Y$&?#i8ysL7ecs>A0~PhOjGF~x#q-l z*!glhnNXSO6iFsWXRSm|&I~ZleZ%HWW#OcX7Ez&{Jd!%I{d`4PM2_iwSp>GBLHbF? zK_k%xcS&4zbF?Phq$gwHz0;fYQcsy!{V$tV^#(eB4NK-Gw0={h7yI@e-wW`z7sB!I z!Ek0W?)M$``{rPVi6+({aJm$c)$IK%$~ei#UUlRdH#cLJ764*GaG|V|pbI*3qnb~p zj;x#>F^(@q=!yLs_VoKjxe^)Oa0O{;&{&?u*;rQPaE=CgMwe>o7N_i$R_4mM>sPzz zq#iP2Dn-gAY|)sqC7`avXGAR?-h8t3BBM6WjJ$i;pKVhDT{k_xn2;+@u>y87eFb3O>`GhQB41u)Oa55_9IbO@~{Xf6Do!f z{0vlil^ei5{)P!W$@um_;9Gtuo9z0st{Fw_=4;yB)4KhVgd}Z|y6mudCqe2Pa{d20 z6C$)t(SHsPrl+iNkO)r{s9#?O=X$4h>>aO%+a#qdfJsRyg|G6nlkOZhX5; zOEUwydsK;-08#)XE1VvQ-IS>Z+>jI2a$1ZGfsqz2;iF<5jVLK5lspQuI9mUtU>+%f zBA=nJ4s!8n255bPSlFl_N7_~)(Pn%(TQr_bA#W0vaYFnor(BcxZ--& za=XLS#*N5Qu@ty(k2B_qCN4TtzUcYnvBwW!#LR{AwG<@y>igGb(S~IkQT0#JqatN% z;0Bn!^M#r1Jo9m>ryW2#xHu;xqmkfa{pF!K%Fb5b#!lbQmE92t&psgzS+l@TeD*ON z!I`R(4JuchdJI}gg7T#O!M~Gl{jsoH=S7ScQ%jP{l6j%X9dW+=wCSU}4-Ak0rs_Qa zSME{bLVTQUxwN#Wu^xfZvbrU>p=6}0cV}zjBCx9k-@_!gX5^?=zdLEau6Qqa=+n8r zV~*adO|cAjcy)$MbH1UB``dD#KXz|jzpIAjTpJH|JUg~(?P}JOD*x>w@aoNPnSL>^ zyP~ez{GD|e*G(Pd2pSZEPP%^-=oGi%ci7B9_94L!`m0+WqmwCQ6JaR-QzsbxUtZ0H zf)U|K^mMLq)b@G#f@8a6az@-g>h_FhnPHfD$Re5nZ5StsXFrwNCU`TV-8=tl-5dID z`V!$~f59WuJBT(H3>FLP$cqU2`{gL;`RL#tQeOKn#8nga&bFN85a6U!{5Mq}0HnMb zEFF~qBs_K-7~J$H)Z&jI3E@AW)a_|!4ouU$LMfoL{1>gq?!YDd;%SM2c9hN zKAj5y5irUnqDem9n>ab8DW!!JLA%g~pF|@LELp-AI#S!=rIf0)fXuZYRThM(Vi2OX zn*A7DuD7sI0|J9Z6>nbS-yEE#x6>w96iI>7NnXAour}&+`&avs;TI!HFL~i&0&6%Q z?W>2jj|`?W80wEDLJ%5{-Iyqy8h#;gK*XVbO>` z38CJP2CWypmz-j&&mLh>4##(umL-kM)u;aS+08SHkEfQljfUeuKegr2eG4H|nHQY{ zN6CYjM-%z0vy#qSnf4{&_SRYGgReq4!DyhAw9^uw@CMj=ZEI^+hX*icT#EzX4*l zv3wC$3-h+)*tdx?0dr-S(!ur1#S^HR zCndBWf$WN-35DL5xD;J#2<$apQN^io6g{=#%28O){bwV2C0oRn=+YFa&xl9-I~^Uq z*j(w7waWYg0kV>In(cPcRNYydfh(8bx~S~eC8)@yDgA>Ii$ktaVhi1uo!Q3q)>U*G zUo}TtbJlg*NW+KeX&bd-!3kqy-WO%Lw|s}>Lf|eH7OGEf+Kln#i(g&sZtv^A9Uv}` z7r*sG5TR0QGfIAtL&}?~{kd4#2~6i$v2mP9I}cg4@np_WoJCX$E>Y5>=~px03hxnS zD%`6?!YL{C&zvL;aXd?u`u-k_k)Y>j^e*@D@ghF=N)p!bIl$3mW+bX8jeg-E&(*bC zSCZ16^>-q*0w}0awuSp{jN`U-&=5u&< zfIssvBJwDMJFg;$0$wV;P&1E+6a;aNwp}HOtSniLbiOeETeSZI2!<`qpEH6neR-iB z>A4Pu;B2YZY$H~Q)>FQQZ6|Y|U%&G&_Rhk2=DVF951c<=f98@ViP|F5gffm-&Jq!M zH&Q1bo0anvbHx_-qjDmvSw~|EDV+|A2Bv`=%ERf1lu|ijR55_beYQfTBMrs7j*C3O zafk_LfcaBfm+=?x&!U;H*4iTxq!sN4HDol>JW!K|8AE3Vh{_dZ-y|@DudX0(mq^f} z3kevE7xS3pG(Y-KNU;2wvutgoPRtS4!j(&8BX~C%9$RdG#B8!!7pJ!p5ZLwV#a|)1 zQcT3DG(CeDeT3~j{j^Zvi5&C2LV-TPLNIYKR)G$ax+ZpZ2_gVF#K76Zx?v&5gn{H0 zZM!5{exX9}PuU0W?U|8XJ81RBebsT8%6{5MCUfSavdgb>j0Hitt;V#&Cx zp2J8SEu=WYCh!v0c8YvY#nV*&?7&{GmgKsjlAMl(~At|XRZBjX4!sY*tVtBuEQ zfea){NVuKh_4CWe&8DY>)^l~N7>f8%$EA*$qFZjEjSpM1-Rli&FI%pbaQ5CZNkmzq zCeJ5!hm$ZAR8neo$P_YYxeS7biL57u%s56|_7FoYZxGn3ls2r!YBi5x%u0ipp@f8$ z%J-9qEY~N6ug=ODBN|2JygUYqN!TTj`TYJ;glb{Fpk4#1g5#P^Z&Fafk2jnT`k9pA!B@SyJYTWAnmT(Tr+vp8?+*+ib zoVfN)W%chuG-3KO_~x%}`_DLHRLitX1OSp2jug_ma#9#>DK*fuGCTr*Y8Z$OTA9J* zekVmvn@U0W#)Eld#M3U{#mCj7aVR!(D%00q$ER|IVt6s5Bxr!7y6%GnNfXiBPzv7! z*Mt)b&O;Et(?QJ3KG!g9{F|YQnv*%Ehy?`;l5iC+Ta%<#T}{$&P3LaB@CEkq8kXa0 zwr|wP-K(z_D^qRDy#SW<4xF=Zz+l43(-+w|D(Exndnd*5Z}c3&F@hgV;R3&ys&(fd z;6%;MNO__@M!omw9G@|fB@ygyvC-{l*6QUA%6kYpb{0{H!6J09wp_~-HPh$g~8l1s2Y_fc*d@(G~-IK zgdat=_P7WYY>23dhU>lXNtI%k)Yx(?hK6w2cF`G)vHpb##{eSf4DHKZ7LZhvlW!Wm zD|iaLg&Xn%UwpsdZHS6IhHokBA*H>5bFJQs4C$uXS2u33#*4g6R3_GmSmzsI%c51E zcv_R4$zHCELIklqhZ2qJcV9QViw={*9kz0=Ivfsn{!P_~VBqd2I1jsp?SbFvIgP|5 zXOb#7mTC62sZgUrx-52H2?ad`LwdLzk)1T=^wr#{EHJc)c>~Xo&);!oH0t%-Q82&Q zqi3`a9|LF(ytN5Xlj%3CbYW;0iMywad%v<=W7RNn%&Eux_M|L>?^c<3DBCADY5Z!^ z#vMgmIa}zb@AoCZJn<1@GEI=PJ0+mM9;1)o zd85wA8=uUeN{*#Z9la??wfhGL*^m~DA?;$|!asZX{oho*g42l$wJ#(v=(Tv@-dz+8 z37?fo`5*@5*hs^#83D|ADoaS+MnP1l7CUryIhh1eyNOHBhKZG%>fAcv6incPj4$xkDDUxKH z2C+#|F>>y^NP5mg%WQHFurglCi;da$trE%CoJRMbH`ct}S6CaKBxb6Ty()|XD@rwt7()qIe~*veR^{OhZ~Cg(`I*hsj(>K9jv%VPt`nUb*!#j= zj$dTkltm>gX64}~2ccjzYdS6%r7SSN{Rh;!4yI`+eRIM4fTZ5+^c?y6 zEh}3J9=+8c$D|C!zx5QfCNWu%gv+?BU;$h78eVDG>qcY_vF6b>&JWtgOi)wqw+n2a z2fIL)$kw7AM0{AN&D?5ZEh%hbOo%QP^@ADVMqdNX5nSFcC8w`zaa@(~<*li@^fP6j zQ3-1zcCz-1sC5jM$IE`%cuXIuq)<QTaW%nc)t}U?tt(1<8g%y zjV?klJ?p~UW(Fb(y{IpaZXV5uM?S{*)7?92$Ht#=-gLO!xE!Pi!`RL$IcT^kgxUG! z4=bZ3lp$LC-linHG#Sc&<|3gjAL?HDlrr3(w@r@TE!=s{cmLilxJdFv`R&~_MM7k6 z1zI|Yo|u-lO#us)SWJptsU?iE{jwA(@$FbVBz_*eU4;V&f`{e-_^$6P;jTynRNO8f z!f-u$we@ksjiLe2U<t^R5F!6yf_FJOhfl%W;$af5~vzuj{b0yiYx;u zb#+r^$5-G6P4QXvS&i0hm0JdMTTsz?IUk8_lZx2Zf)9zgPCCJK(hh3l+0|;?2gg=A zD`VBOvvP0SloO@BxHLMc6w$`8JR0Sp;d#&f$!Tm6Dy7SCCGK%7m-~I}#DLc{0oqee z0Y2fadM{Vc6~4+g>iE>OoI}Y(i1P%ECZK*Y9j0zwznV=N#rgFN0}qJ>c*d$4 zntR^kyc7g<)3MhHsALcyNX_WpQ#xx8TN0RK;Hprr#oKptCcrd~Z5|Cfqk~?ko3%`= zWr2_*Uf#7QE>(id#p@47B$042IyY@PZzg%}532IlIbNGv7wr)e)=p@~?L0`zuG5iJ z2p_Qm2k`r8375-+6#M|Ak;nt@0kPPza?XBvq9~yUY;Pt1EhD$V_*?#_Q{rP2Z+V;^ zqDslMmpB=CtWo%8`kE<{`7mou5&bafW#9aHvt05;FKdrA(;G8#Ex8%s8|oM&`NuP9 z`K>LF4S8sRQbiyk&8f_!CDw~;a|L$B=NyDeeAhYMk&+^O>Ctj8zO3m6J~kTiji~aVXJf%+aNw)(gE~k%c!(@EgQi#|2oZn-3 z-k}fAIh3&nsW)GqzB_LzbnIDXT+jt|#X%9Q!Ko-eZLe5Z;w>#pPiPp>00nz`O$WVB z5O#6TgL(VE({bRMtohT@V}`FMF(?NP_zRuPC~J_T_&?jX?{^O_l00AGx4f7Q_(25* z?;fMg$7Qu>_w*{`Y?7Gb=r4tf8Dtg$;Nd%&BcZ*6D|xf(Wm`0=ysbff`G8>1}WeV*AaI$gw<>}*VtS2nQPB{gW`lK8b4 z?O@y2+0Noqxm4&3)RIR#x*=H;D4ebb>-yOJH<`)wywaZ&Wn;Bq*+NQ1$QiXhgk8`g zjH}-AM5}}VBK0A)is)RYpc)IU&t95T^`L@G1ud~Ob{7Fc?3WDU^tN5<0-lFfIBHE16dSaE9Rj38a)$9I=bA#&(!nMf&#Gf4HYrc9<5Y<4R z(=#Q2dn&T4Xh2AGpbUR-MVPIIqgb}q&@xg(vf&5@AT~?lLAVe?_FCS@9QIapI8Rs) zX@1~fRy>i{R7aK%y@uOhjo_?|{0&B`nYv`hk)GLXeK22A< zKrjty2dqvjkD(r@976&GY7{3hKleT>j5`!#amXQjmvhMeIF+jWcHgZ0EU4!SRC?1i z3XisXiyJ)Sl39wANl8)TqHuXs>DKv~zOV;zMb- zBfTJ8v;^dHp6#>x81IsYyOh`X*(@P-jD$Q+DP_VRWy}<7MUwl;WGFV?ldwvtf+ISQ zVQVy4ZPEZS+;|$$Zw#wZ9Ac`0VIl3Bdj-!0@Gp2M50|?9XUh!i$a-^8&@l0;`s_m{ z-6!UnH;_mwi}$S+4waAzpDXj4nR-4BN4dVcuf`*tb^C=cXC6$@ttlueC})xJT&?~g zQ7J|362&N6Sg`SXd2YE$Jnx+sx%Kb-MtrFYC2n}=#(fa>>)L|{5`00Kth0K=hL$WSTOT zoJh?wmw$^h*3vwWZ7eNTKwEao9yEd0*TZGAwp~^(?9{#GH z81sQsPALxRi=COK0^^$8cg#fd*)Pmau}dkpL9neIxTQQiUIYN|-Qn>fniWNLWp{Yt zbfU2pMl$^x)PYJFs!3hv3ymkI$FTKOT zqB{@MtAug&-LBrpTpyaApxqOiuvMT=rEHSO_TW>)dWP?&s^eJ({redMHx-qCfRoMB za6^xS+;>jb(ns@9-@Eo722CpkT~pyM-nHjTF85hE{sON*rSqnE49i<^cx{)dc%$#G zS<+s;#=FEEutmnjLN5F6aT9#G{wr0HJ~rJO9g%fWE&s26duEs!Zrop(wx(5lVJA32 zCq*W@cJCwimrIV?4Bz%7__MDw3`s&KxkxboMa#v(-#;Y+~E@mkZXc`W9q)OO+ zOZ|uUEcyfEMcIKDN?umb%Tg+0z|b7-P~uK6S)Vqx@>PPB=gw7EQ{mV&f?A)Z-4TP_ zXV1Tn>!U>zlWdwE4l$<-1Cl_4I$X(@oH}+GroY3#Mz{_JevlQpAa{p5ij+M^(w@GJ zR)7$CjS+wMH2e`F-RF1`e4xTU@tH!2OPx&T1{00$AT3UXAt7D#j`$V%*#S;BSS1H* zpD?Ny)y;FA-}1BZ`R{KmmW-YSKbdIC`m8uqpgotrH$J(q|N02Ee5X(7nlhy=FeGTI zF*2@{lL7SczY5?0#!h3G**TD;tI>+LAA$C~nbF#L?8`ye-TH-q{C3rng=Ge}8R{EC z#jkQ%;_uB)=hZlgFsu;%E)NUStV5t&vsFfBG zjGPUDEFA3OW}(5A%1*Hl$E4Stm)D#I%Eh5Q)>xw zV}~N^h=$$w(?uazC6d-wINsSob-5QkT{!)|#-u4#uW6}!tF)zNws{t>8K6v(z9Pj* z7@!ELa6w--pGg@MPU4`#j%Ivl*}f!hr3{(%61;7B#Cq@&TbEmo7+BxmsnUnpx7t^( zzS{9`Ml2^MMu}%IDv-}5rnJXDMFN<2rcKhV8T*&l{~qh3pRDpalh9yEs*>hM35g)k z_yT*z4?D)s)cXDFW`QGRAC&F*C&7Gew$r3k(-1DW@lON|cZ-GXf^i-*4|MS1D^CT* z{XU1s1@_drO`Aypy!z*Dd2(&Y?#K)8ZD=%PEXQ0G?jO65LfUDpwSqRE4h9yTr-mjV{|wNz^K z9iI{K^{_bM1(i*9rMXl}XP8Kb?o}|WsQk@J{S^2LHL-lZD7~;SjdqvdGqp7pH)N?( zEe)4i=%shsS#K0?v@B+%+j=Q>#;~Mu)Ws$^{(D&^1dMVebAq}&d697D1hE;6ijcF) zA!xy-$Q|_NWk(WD=P9w3NiD8bv4{uFR)Cl=k^Uosb_c-aC1klM@Q!IHD)9}&6}E^) z3^Ygm*z7dbnbAFHzw&jGU13lk%RTU$Xw&76de0hDQ;@Y3@~C{I&OE@~Pz7;;oh;3! zmM~unGT-)PnT>Llfy#?Bu{aKqWECB$tW?^jNXVlUb!=sD`NFcXOcwx~hnWY>ve^2r z2;?VBIs*W~9#ezJkus}P$SewF7t%U&6XcoddT9I>7Y(#swu1bINCi?U0wUE&oLoVV z9}`H+%hNZu54*k^40R0rvD2(EDAvS2iG@aYxsm~Be~2{UdU3D=wZrNB7}ERC+|mfe z5Gw+z#9K`=oi8^9)>(QrR$w1zmrlIy!2Q!?}I{Y=DZ5Z-1ZdH6X zx3_4MYI~Yt3CKg%C7y!mJH$DV>ERVLD zi^MKo!AiG8s3Ybfm$Mccgzimlwo@uA72TwrvqHBHUv}BmGVMy?mFJ=wPIq6a20^6G zP%v7l3YtQk-Y8*7PHV$Dv?dZcRIx}3z?#JO@AupTu$n`ts~Md_*s=HT!x-x+Ig-B1 zxvbWFx!BR*9SkQAMR$g%yzxID0eyjWEuPJv9b9ZCcoeXk01(WejO_S^Rw&~FE(a7% zBswx=KqfzbIej@eR{l%xbI{at<=V2jkwe9-;+3o3<$P?-JL<`I#UOmnXD@#wkrjX ziQuKHSPl?*;MzUnA@=l2J?d}Z>q0i{e_+uU0ES5bWi_%*-!b-qL~wjN!gvX-3B)4N z5{VV2ZLFGoZj|1E!cxZRjpCw3#wjQc6hun^0LcIpQRv8?2=oZjk;ru623ijrkI8+A zd4B+?e)f`U>0MkVQ73o#Bvz4Bgb^CKjt*RCWW}wzS0?r^8e4|HcvMa@63qleO^2tT@Cj(>l0 z_tpk!y8hiA&SbP69(>&vbZ4;J=?127XebiBqRTR9RQSr#)~-S1WVZat3IizE%v&-P z`0aXcM|}Z;`v5@N82W$|QrK1#G)z2*&QE+y0yQRM+bxk{o0=vcS8Q>)s?q(>`|_rE z^H*7wyZ%omd}{O6>ot5kdNxY=IsBqpY%1XoDy#`MJJpS#es1xkcQN+*#`hUyfb2M_!dVwo zs1lxvp~Sid04OgBI89RKKm07sf{pbChq}#dvcd#Th37cYPxs1U_G@P^FgR|GXXqC| zUO4PkDY9+9aGaIRkLb5SFqpY6L7MUi>yOs>=Jhpyiu_`SiG*biLGCg!>}V7ka5w@o z>`_xqT3w!>iEt^5~kSy@h#kR{k6gaoIkq98%|zkTEX--my*{sExM^oEcn zUaH@hcaLl?{?Grv>imzN40{B^^`jI70*K4QEdU8vdleQ5Jpbdn{=YZ)_nfP+5oB_F WW^#Hsf 0 + + # MP3 files start with these magic bytes + # ID3 tag or MPEG sync word + assert binary_content[:3] == b"ID3" or binary_content[:2] == b"\xff\xfb" or binary_content[:2] == b"\xff\xf3" + + # Write to file + response.stream_to_file(speech_file_path) + + # Verify file was created and has content + assert speech_file_path.exists() + assert speech_file_path.stat().st_size > 0 + + print(f"Azure TTS audio saved to: {speech_file_path}") + + # assert response cost is greater than 0 + print("Response cost: ", response._hidden_params["response_cost"]) + assert response._hidden_params["response_cost"] > 0 + + except Exception as e: + pytest.fail(f"Test failed with exception: {str(e)}") + + @pytest.mark.asyncio async def test_azure_ava_tts_with_custom_voice(): """ diff --git a/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py b/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py new file mode 100644 index 0000000000..277a47a03b --- /dev/null +++ b/tests/test_litellm/llms/runwayml/test_text_to_speech_transformation.py @@ -0,0 +1,67 @@ +""" +Test RunwayML text-to-speech transformation +""" +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.llms.runwayml.text_to_speech.transformation import ( + RunwayMLTextToSpeechConfig, +) + + +def test_openai_voice_mapping_to_runwayml(): + """ + Test that OpenAI voice names are correctly mapped to RunwayML preset IDs + """ + config = RunwayMLTextToSpeechConfig() + + # Test OpenAI voice mappings + openai_to_runway = { + "alloy": "Maya", + "echo": "James", + "fable": "Bernard", + "onyx": "Vincent", + "nova": "Serene", + "shimmer": "Ella", + } + + for openai_voice, expected_runway_voice in openai_to_runway.items(): + mapped_voice, mapped_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={}, + voice=openai_voice, + drop_params=False, + kwargs={}, + ) + + assert mapped_voice is None + assert "runwayml_voice" in mapped_params + assert mapped_params["runwayml_voice"]["type"] == "runway-preset" + assert mapped_params["runwayml_voice"]["presetId"] == expected_runway_voice + + +def test_runwayml_native_voice_passthrough(): + """ + Test that RunwayML native voice names are passed through correctly as-is + """ + config = RunwayMLTextToSpeechConfig() + + # Test various RunwayML native voices + runway_voices = ["Bernard", "Maya", "Arjun", "Serene", "Chad"] + + for runway_voice in runway_voices: + mapped_voice, mapped_params = config.map_openai_params( + model="eleven_multilingual_v2", + optional_params={}, + voice=runway_voice, + drop_params=False, + kwargs={}, + ) + + assert mapped_voice is None + assert "runwayml_voice" in mapped_params + assert mapped_params["runwayml_voice"]["type"] == "runway-preset" + assert mapped_params["runwayml_voice"]["presetId"] == runway_voice +