diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1-py3-none-any.whl new file mode 100644 index 0000000000..2da8f74acd Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1.tar.gz new file mode 100644 index 0000000000..6e2e47fa7b Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.1.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101123718_add_vector_store_index_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql similarity index 53% rename from litellm-proxy-extras/litellm_proxy_extras/migrations/20251101123718_add_vector_store_index_table/migration.sql rename to litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql index 8b9048002e..af13500d1c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101123718_add_vector_store_index_table/migration.sql +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20251101131415_add_managed_vector_store_index_table/migration.sql @@ -1,5 +1,5 @@ -- CreateTable -CREATE TABLE "LiteLLM_IndexTable" ( +CREATE TABLE "LiteLLM_ManagedVectorStoreIndexTable" ( "id" TEXT NOT NULL, "index_name" TEXT NOT NULL, "litellm_params" JSONB NOT NULL, @@ -9,9 +9,9 @@ CREATE TABLE "LiteLLM_IndexTable" ( "updated_at" TIMESTAMP(3) NOT NULL, "updated_by" TEXT, - CONSTRAINT "LiteLLM_IndexTable_pkey" PRIMARY KEY ("id") + CONSTRAINT "LiteLLM_ManagedVectorStoreIndexTable_pkey" PRIMARY KEY ("id") ); -- CreateIndex -CREATE UNIQUE INDEX "LiteLLM_IndexTable_index_name_key" ON "LiteLLM_IndexTable"("index_name"); +CREATE UNIQUE INDEX "LiteLLM_ManagedVectorStoreIndexTable_index_name_key" ON "LiteLLM_ManagedVectorStoreIndexTable"("index_name"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index ac1acee908..605a312db4 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -590,6 +590,17 @@ model LiteLLM_SSOConfig { updated_at DateTime @updatedAt } +model LiteLLM_ManagedVectorStoreIndexTable { + id String @id @default(uuid()) + index_name String @unique + litellm_params Json + index_info Json? + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? +} + // Cache configuration table model LiteLLM_CacheConfig { id String @id @default("cache_config") diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index d49db2a9a8..0707775939 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.3.0" +version = "0.4.1" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.3.0" +version = "0.4.1" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 8fa65002d5..179bd7aeff 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -83,10 +83,12 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): """ vertex_location = self.get_vertex_ai_location(litellm_params) vertex_project = self.get_vertex_ai_project(litellm_params) - engine_id = litellm_params.get("vertex_app_id") collection_id = ( litellm_params.get("vertex_collection_id") or "default_collection" ) + datastore_id = litellm_params.get("vector_store_id") + if not datastore_id: + raise ValueError("vector_store_id is required") if api_base: return api_base.rstrip("/") @@ -94,7 +96,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): return ( f"https://discoveryengine.googleapis.com/v1/" f"projects/{vertex_project}/locations/{vertex_location}/" - f"collections/{collection_id}/engines/{engine_id}/servingConfigs/default_config" + f"collections/{collection_id}/dataStores/{datastore_id}/servingConfigs/default_config" ) def transform_search_vector_store_request( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index aa442144a8..1f57cd9f15 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -7,18 +7,9 @@ model_list: model: openai/text-embedding-3-large vector_store_registry: - - vector_store_name: "milvus-knowledgebase" + - vector_store_name: "vertex-ai-litellm-website-knowledgebase" litellm_params: - vector_store_id: "my-collection-name" - custom_llm_provider: "milvus" - api_key: os.environ/MILVUS_API_KEY - api_base: https://your-milvus-instance.milvus.io - litellm_embedding_model: "azure/text-embedding-3-large" - litellm_embedding_config: - api_base: https://your-endpoint.cognitiveservices.azure.com/ - api_key: os.environ/AZURE_API_KEY - api_version: "2025-09-01" - milvus_text_field: "book_intro" - # Optional Milvus parameters - annsField: "book_intro_vector" - limit: 10 \ No newline at end of file + vector_store_id: "litellm-docs_1761094140318" + custom_llm_provider: "vertex_ai/search_api" + vertex_project: "test-vector-store-db" + vertex_location: "global" \ No newline at end of file diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 5d1c2febad..cd8f6c3800 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -1504,6 +1504,7 @@ async def _base_vertex_proxy_route( endpoint=endpoint, target=target, custom_headers=headers, + is_streaming_request=is_streaming_request, ) # dynamically construct pass-through endpoint based on incoming path try: @@ -1511,7 +1512,6 @@ async def _base_vertex_proxy_route( request, fastapi_response, user_api_key_dict, - stream=is_streaming_request, # type: ignore ) except ProxyException as e: if headers_passed_through: diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 39187edb52..b66bc6eac2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -995,6 +995,7 @@ def create_pass_through_route( include_subpath: Optional[bool] = False, cost_per_request: Optional[float] = None, custom_llm_provider: Optional[str] = None, + is_streaming_request: Optional[bool] = False, ): # check if target is an adapter.py or a url from litellm._uuid import uuid @@ -1115,7 +1116,7 @@ def create_pass_through_route( forward_headers=cast(Optional[bool], param_forward_headers), merge_query_params=cast(Optional[bool], param_merge_query_params), query_params=final_query_params, - stream=stream, + stream=is_streaming_request or stream, custom_body=final_custom_body, cost_per_request=cast(Optional[float], param_cost_per_request), custom_llm_provider=custom_llm_provider, diff --git a/poetry.lock b/poetry.lock index c252b00763..ab34c86351 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2645,13 +2645,13 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.3.0" +version = "0.4.1" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ - {file = "litellm_proxy_extras-0.3.0-py3-none-any.whl", hash = "sha256:5c366ecb15e9209908cd6142c2d2fc3d0e95b5e3115c9601cc192acc2d444821"}, - {file = "litellm_proxy_extras-0.3.0.tar.gz", hash = "sha256:d72993f88e8b553238081f7b1704c49d856bd58c7737838903614b15de479a8d"}, + {file = "litellm_proxy_extras-0.4.1-py3-none-any.whl", hash = "sha256:9f9b29c9824fe153a5cce5001211884fab858a47185bc7fcda39e5fca07c7b9b"}, + {file = "litellm_proxy_extras-0.4.1.tar.gz", hash = "sha256:5d055352e0592d062604fdd4067294e52b0692575cef140c09b11b3cc633169a"}, ] [[package]] @@ -6519,4 +6519,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "2fd4bde64ab8eab79338d2abc977197a22479ae6ef8510434d28cc49bb1213ab" +content-hash = "4aab4263275116fad0eaefa44fb10d8a9934525bf506bd35db39ca9310822691" diff --git a/pyproject.toml b/pyproject.toml index fd7e9f46d1..6b219307ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ websockets = {version = "^13.1.0", optional = true} boto3 = {version = "1.36.0", optional = true} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "^1.10.0", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.3.0", optional = true} +litellm-proxy-extras = {version = "0.4.1", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.20", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index f2e31d45ab..ce1c41ce26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests cryptography==44.0.1 tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.3.0 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.1 # for proxy extras - e.g. prisma migrations ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env tiktoken==0.8.0 # for calculating usage diff --git a/schema.prisma b/schema.prisma index a142b388b2..605a312db4 100644 --- a/schema.prisma +++ b/schema.prisma @@ -590,7 +590,7 @@ model LiteLLM_SSOConfig { updated_at DateTime @updatedAt } -model LiteLLM_IndexTable { +model LiteLLM_ManagedVectorStoreIndexTable { id String @id @default(uuid()) index_name String @unique litellm_params Json