diff --git a/litellm/llms/hosted_vllm/embedding/README.md b/litellm/llms/hosted_vllm/embedding/README.md index f82b3c77a6..2c58e16fc2 100644 --- a/litellm/llms/hosted_vllm/embedding/README.md +++ b/litellm/llms/hosted_vllm/embedding/README.md @@ -2,4 +2,15 @@ No transformation is required for hosted_vllm embedding. VLLM is a superset of OpenAI's `embedding` endpoint. -To pass provider-specific parameters, see [this](https://docs.litellm.ai/docs/completion/provider_specific_params) \ No newline at end of file +## `encoding_format` + +For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request: + +1. Explicit value on the embedding call (`encoding_format=...`). +2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry). +3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env). +4. Default **`float`**. + +That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly. + +To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params). \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 0079bd750c..0553cf9d42 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4923,8 +4923,17 @@ def embedding( # noqa: PLR0915 if encoding_format is not None: optional_params["encoding_format"] = encoding_format else: - # Omiting causes openai sdk to add default value of "float" - optional_params["encoding_format"] = None + env_fmt = get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT") + if env_fmt is not None and env_fmt.strip().lower() == "none": + optional_params.pop("encoding_format", None) + else: + _default_fmt = ( + optional_params.get("encoding_format") or env_fmt or "float" + ) + if _default_fmt.strip().lower() == "none": + optional_params.pop("encoding_format", None) + else: + optional_params["encoding_format"] = _default_fmt api_version = None diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index 3f1a397ebc..82510b6f4f 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -1257,22 +1257,13 @@ def test_jina_ai_img_embeddings(input_data, expected_payload_input): assert sent_data["input"] == expected_payload_input -def test_encoding_format_none_not_omitted_from_openai_sdk(): +def test_encoding_format_defaults_to_float_for_openai_sdk(monkeypatch): """ - Test that encoding_format=None is explicitly sent to OpenAI SDK. + When encoding_format is not provided, LiteLLM sends `float` for OpenAI-path embeddings. - This test verifies that when encoding_format is not provided by the user, - liteLLM explicitly sets it to None rather than omitting it. This prevents - the OpenAI SDK from adding its default value of 'base64'. - - Without this fix: - - OpenAI SDK adds encoding_format='base64' as default when parameter is missing - - This causes issues with providers that don't support encoding_format (like Gemini) - - With this fix: - - encoding_format=None is explicitly passed - - OpenAI SDK respects the explicit None and doesn't add defaults + Optional global override: `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT`. """ + monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False) with patch( "litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client" ) as mock_get_client: @@ -1310,17 +1301,12 @@ def test_encoding_format_none_not_omitted_from_openai_sdk(): call_kwargs = call_args[1] # Get kwargs - # The key assertion: encoding_format should be in the request with value None - # This prevents OpenAI SDK from adding its default 'base64' value - assert "encoding_format" in call_kwargs, ( - "encoding_format should be explicitly passed to OpenAI SDK " - "(even if None) to prevent SDK from adding default value" - ) + assert "encoding_format" in call_kwargs assert ( - call_kwargs["encoding_format"] is None - ), "encoding_format should be None when not provided by user" + call_kwargs["encoding_format"] == "float" + ), "encoding_format should default to float when not provided by user" - print("✅ PASS: encoding_format=None is correctly passed to OpenAI SDK") + print("✅ PASS: encoding_format='float' is correctly passed to OpenAI SDK") def test_encoding_format_explicit_value_preserved(): diff --git a/tests/test_litellm/test_openai_embedding_encoding_format_default.py b/tests/test_litellm/test_openai_embedding_encoding_format_default.py new file mode 100644 index 0000000000..94e4e3c81e --- /dev/null +++ b/tests/test_litellm/test_openai_embedding_encoding_format_default.py @@ -0,0 +1,124 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from litellm import embedding + + +@pytest.mark.parametrize( + "set_env, env_value, expected", + [ + (False, None, "float"), + (True, "base64", "base64"), + ], +) +def test_openai_embedding_encoding_format_default( + monkeypatch, set_env, env_value, expected +): + monkeypatch.delenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", raising=False) + if set_env: + monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_value) + + mock_response = MagicMock() + mock_response.parse.return_value = MagicMock( + model_dump=lambda: { + "data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}], + "model": "text-embedding-ada-002", + "object": "list", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + ) + mock_response.headers = {} + + with patch( + "litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client" + ) as mock_get_client: + mock_client_instance = MagicMock() + mock_get_client.return_value = mock_client_instance + mock_client_instance.embeddings.with_raw_response.create.return_value = ( + mock_response + ) + + embedding( + model="text-embedding-ada-002", + input="Hello world", + ) + + call_kwargs = ( + mock_client_instance.embeddings.with_raw_response.create.call_args[1] + ) + assert call_kwargs["encoding_format"] == expected + + +@pytest.mark.parametrize("env_none", ["none", "NONE", " none "]) +def test_openai_embedding_encoding_format_env_none_omits_param( + monkeypatch, env_none +): + """LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT=none omits encoding_format (provider default).""" + monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", env_none) + + mock_response = MagicMock() + mock_response.parse.return_value = MagicMock( + model_dump=lambda: { + "data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}], + "model": "text-embedding-ada-002", + "object": "list", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + ) + mock_response.headers = {} + + with patch( + "litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client" + ) as mock_get_client: + mock_client_instance = MagicMock() + mock_get_client.return_value = mock_client_instance + mock_client_instance.embeddings.with_raw_response.create.return_value = ( + mock_response + ) + + embedding( + model="text-embedding-ada-002", + input="Hello world", + ) + + call_kwargs = ( + mock_client_instance.embeddings.with_raw_response.create.call_args[1] + ) + assert "encoding_format" not in call_kwargs + + +def test_openai_embedding_encoding_format_explicit_overrides_env(monkeypatch): + """Request `encoding_format` wins over LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT.""" + monkeypatch.setenv("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT", "float") + + mock_response = MagicMock() + mock_response.parse.return_value = MagicMock( + model_dump=lambda: { + "data": [{"embedding": [0.1, 0.2, 0.3], "index": 0}], + "model": "text-embedding-ada-002", + "object": "list", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + ) + mock_response.headers = {} + + with patch( + "litellm.llms.openai.openai.OpenAIChatCompletion._get_openai_client" + ) as mock_get_client: + mock_client_instance = MagicMock() + mock_get_client.return_value = mock_client_instance + mock_client_instance.embeddings.with_raw_response.create.return_value = ( + mock_response + ) + + embedding( + model="text-embedding-ada-002", + input="Hello world", + encoding_format="base64", + ) + + call_kwargs = ( + mock_client_instance.embeddings.with_raw_response.create.call_args[1] + ) + assert call_kwargs["encoding_format"] == "base64"