From bbeb094d00ee890850996e012c5c7490e5b04fee Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 14 May 2026 02:39:12 +0530 Subject: [PATCH] Litellm agent oss staging 05 11 2026 (#27733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ollama): Include provider in model list for ollama (#26135) * Include provider in model names for ollama * Fix unit tests * fix(ollama): process both thinking and content in same streaming chunk (#26098) * fix(health_check): skip max_tokens for image_generation mode (#26417) * fix(health_check): skip max_tokens for image_generation mode `_update_litellm_params_for_health_check` injected `max_tokens` for every deployment. OpenAI `/v1/images/generations` strictly rejects unknown fields, so health checks for dall-e-* and gpt-image-1 always failed with `400 "Unknown parameter: 'max_tokens'"` even though the actual image endpoint calls succeed. Skip the `max_tokens` injection when `model_info.mode == "image_generation"`. `messages` still gets injected (downstream `_filter_model_params` already strips it for non-chat handlers). * Switch to allow-list with per-deployment override Per @krrishdholakia review: deny-listing image_generation only re-introduces the same bug for every other non-chat mode (embedding, audio_*, rerank, video_generation, ocr, search, moderation, ...). Replace the single image_generation skip with `_MAX_TOKEN_SUPPORT_MODES = {chat, completion, responses}`. Missing `mode` is treated as chat for backward compatibility. New modes are safe by default. Add `model_info.health_check_supports_max_tokens` as an operator escape hatch — True forces injection on a non-listed deployment (operator wants to bound probe tokens), False suppresses it on a chat-style deployment behind a strict-schema provider. Tests: parametrize over 3 chat-style + 10 non-chat modes, plus override on/off and the no-mode legacy path. * fix(http_handler): handle RequestNotRead in MaskedHTTPStatusError for multipart uploads (#26718) Squash-merged by litellm-agent from dawidkulpa's PR. * fix(ollama): guard against double 'ollama/' prefix in live model listing Greptile flagged that Ollama servers can return names that already start with 'ollama/'. Check the prefix before prepending so we don't produce 'ollama/ollama/...'. Adds a regression test. * Fix Ollama empty reasoning stream chunks Co-authored-by: Yassin Kortam --------- Co-authored-by: James Myatt Co-authored-by: VHash <225398745+vhash0@users.noreply.github.com> Co-authored-by: hayden Co-authored-by: dawidkulpa <84176950+dawidkulpa@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: Claude Co-authored-by: Yassin Kortam --- litellm/llms/custom_httpx/http_handler.py | 7 +- litellm/llms/ollama/chat/transformation.py | 4 +- litellm/llms/ollama/common_utils.py | 2 +- litellm/proxy/health_check.py | 39 +++++- .../test_credential_leak_prevention.py | 25 ++++ .../ollama/test_ollama_chat_transformation.py | 65 +++++++++ .../llms/ollama/test_ollama_model_info.py | 30 ++++- .../proxy/test_health_check_max_tokens.py | 126 ++++++++++++++++++ 8 files changed, 289 insertions(+), 9 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index af18c66667..e11d8532db 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -485,11 +485,16 @@ class MaskedHTTPStatusError(httpx.HTTPStatusError): if k.lower() not in ("content-encoding", "content-length") } + try: + request_content = original_error.request.content + except httpx.RequestNotRead: + request_content = b"" + masked_request = httpx.Request( method=original_error.request.method, url=masked_url, headers=original_error.request.headers, - content=original_error.request.content, + content=request_content, ) super().__init__( diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 48534799c9..e36150a495 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -507,10 +507,10 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): # PROCESS REASONING CONTENT reasoning_content: Optional[str] = None content: Optional[str] = None - if chunk["message"].get("thinking") is not None: + if chunk["message"].get("thinking"): reasoning_content = chunk["message"].get("thinking") self.started_reasoning_content = True - elif chunk["message"].get("content") is not None: + if chunk["message"].get("content"): if ( self.started_reasoning_content and not self.finished_reasoning_content diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 8aedd9b350..8ca8b7d383 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -108,7 +108,7 @@ class OllamaModelInfo(BaseLLMModelInfo): continue nm = entry.get("name") or entry.get("model") if isinstance(nm, str): - names.add(nm) + names.add(nm if nm.startswith("ollama/") else f"ollama/{nm}") except Exception as e: verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}") # If tags endpoint fails, fall back to static list diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 585ba88394..4a28143e61 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -36,6 +36,31 @@ ADMIN_ONLY_HEALTH_DISPLAY_PARAMS = ("api_base", "api_version") MINIMAL_DISPLAY_PARAMS = ["model", "mode_error"] +# Modes whose health-check probe is a chat-style completion call and +# therefore accept `max_tokens`. Other modes (embedding, image_generation, +# audio_*, rerank, video_generation, ocr, search, moderation, ...) hit +# endpoints that reject unknown fields with 400 "Unknown parameter: +# 'max_tokens'". Allow-list so new modes are safe by default. +# Per-deployment override: `model_info.health_check_supports_max_tokens`. +_MAX_TOKEN_SUPPORT_MODES: frozenset = frozenset({"chat", "completion", "responses"}) + + +def _should_inject_health_check_max_tokens(model_info: dict) -> bool: + """ + Whether the health-check probe should include `max_tokens`. + + Order: + 1. `model_info.health_check_supports_max_tokens` (operator override). + 2. `_MAX_TOKEN_SUPPORT_MODES`. Missing `mode` is treated as `chat` + for backward compatibility. + """ + explicit = model_info.get("health_check_supports_max_tokens") + if explicit is not None: + return bool(explicit) + mode = model_info.get("mode") or "chat" + return mode in _MAX_TOKEN_SUPPORT_MODES + + # Health-check modes that forward `reasoning_effort` to the provider (chat-style calls). _HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT = frozenset( (None, "chat", "completion") @@ -389,14 +414,22 @@ def _update_litellm_params_for_health_check( Update the litellm params for health check. - gets a short `messages` param for health check + - adds a bounded `max_tokens` when the deployment is a chat-style mode + (`chat`, `completion`, `responses`) or the operator explicitly opts in + via `model_info.health_check_supports_max_tokens`. Non-chat endpoints + (image, embedding, audio_*, rerank, video, ocr, search, moderation, ...) + reject unknown fields with 400 "Unknown parameter: 'max_tokens'". - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID """ litellm_params["messages"] = _get_random_llm_message() - _resolved_max_tokens = _resolve_health_check_max_tokens(model_info, litellm_params) - if _resolved_max_tokens is not None: - litellm_params["max_tokens"] = _resolved_max_tokens + if _should_inject_health_check_max_tokens(model_info): + _resolved_max_tokens = _resolve_health_check_max_tokens( + model_info, litellm_params + ) + if _resolved_max_tokens is not None: + litellm_params["max_tokens"] = _resolved_max_tokens # Per-model reasoning effort for health checks only (e.g. reasoning_effort=none). if model_info.get("mode", None) in _HEALTH_CHECK_MODES_SUPPORTING_REASONING_EFFORT: diff --git a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py index 72b4da7b38..0a3bf403bf 100644 --- a/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py +++ b/tests/test_litellm/llms/custom_httpx/test_credential_leak_prevention.py @@ -104,6 +104,31 @@ class TestMaskedHTTPStatusError: # The attached request must be the masked one, not the original. assert "KEY_X" not in str(req.url) + def test_handles_streaming_request_content(self): + """MaskedHTTPStatusError must not crash when request body is streamed.""" + streaming_request = httpx.Request( + "POST", + "https://api.openai.com/v1/images/edits?key=SECRET_KEY", + stream=httpx.ByteStream(b"multipart-data"), + ) + response = httpx.Response( + 400, + request=streaming_request, + content=b'{"error": "bad request"}', + ) + orig = httpx.HTTPStatusError( + message="400 Bad Request", + request=streaming_request, + response=response, + ) + + masked = MaskedHTTPStatusError(orig) + + assert masked.status_code == 400 + assert masked.response.status_code == 400 + assert masked.response.request is not None + assert "SECRET_KEY" not in str(masked.request.url) + def test_strips_content_encoding_to_avoid_double_decode(self): """If the upstream response declared Content-Encoding (e.g. gzip), the rebuilt Response must not carry that header over — otherwise httpx diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 05b96b8822..906c51d806 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -694,6 +694,71 @@ class TestOllamaReasoningContentStreaming: # reasoning_content is not set when there's no thinking in the chunk assert getattr(result2.choices[0].delta, "reasoning_content", None) is None + def test_thinking_and_content_in_same_chunk(self): + """ + Test that a chunk containing both thinking and content preserves both fields. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "model": "deepseek-r1", + "message": { + "role": "assistant", + "thinking": "Let me reason first.", + "content": "Final answer.", + }, + "done": False, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.reasoning_content == "Let me reason first." + assert result.choices[0].delta.content == "Final answer." + + def test_streaming_chunks_ignore_inactive_empty_reasoning_fields(self): + """ + Test that Ollama chunks with inactive empty fields stay in the active delta. + """ + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "model": "deepseek-r1", + "message": { + "role": "assistant", + "thinking": "Let me reason first.", + "content": "", + }, + "done": False, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.reasoning_content == "Let me reason first." + assert result.choices[0].delta.content is None + assert iterator.finished_reasoning_content is False + + content_chunk = { + "model": "deepseek-r1", + "message": { + "role": "assistant", + "thinking": "", + "content": "Final answer.", + }, + "done": False, + } + + result = iterator.chunk_parser(content_chunk) + + assert getattr(result.choices[0].delta, "reasoning_content", None) is None + assert result.choices[0].delta.content == "Final answer." + assert iterator.finished_reasoning_content is True + def test_think_tags_in_content(self): """ Test that tags embedded in content are properly parsed. diff --git a/tests/test_litellm/llms/ollama/test_ollama_model_info.py b/tests/test_litellm/llms/ollama/test_ollama_model_info.py index 95fc80b7fd..448a26bafe 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_model_info.py +++ b/tests/test_litellm/llms/ollama/test_ollama_model_info.py @@ -73,7 +73,7 @@ class TestOllamaModelInfo: info = OllamaModelInfo() models = info.get_models() # Only 'alpha' and 'zeta' should be returned, sorted alphabetically - assert models == ["alpha", "zeta"] + assert models == ["ollama/alpha", "ollama/zeta"] # Ensure correct endpoint was called assert calls and calls[0].endswith("/api/tags") assert call_headers and call_headers[0] == {} @@ -122,7 +122,7 @@ class TestOllamaModelInfo: monkeypatch.setattr(httpx, "get", mock_get) info = OllamaModelInfo() models = info.get_models() - assert models == ["m1", "m2"] + assert models == ["ollama/m1", "ollama/m2"] def test_get_models_fallback_on_error(self, monkeypatch): """ @@ -139,6 +139,32 @@ class TestOllamaModelInfo: # Default static ollama_models is ['llama2'], so expect ['ollama/llama2'] assert models == ["ollama/llama2"] + def test_get_models_no_double_prefix(self, monkeypatch): + """ + Names that already carry the 'ollama/' prefix (or are returned by an + Ollama server that's been configured to emit them) should not be + prefixed a second time. + """ + sample = { + "models": [ + {"name": "ollama/already-prefixed"}, + {"name": "fresh"}, + {"name": "hf.co/Qwen/Qwen3-14B:latest"}, + ] + } + + def mock_get(url, headers): + return DummyResponse(sample, status_code=200) + + monkeypatch.setattr(httpx, "get", mock_get) + info = OllamaModelInfo() + models = info.get_models() + assert models == [ + "ollama/already-prefixed", + "ollama/fresh", + "ollama/hf.co/Qwen/Qwen3-14B:latest", + ] + class TestOllamaGetModelInfo: """Tests for OllamaConfig.get_model_info() api_base threading and graceful fallback.""" diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 72d77862b5..5cb7cdacc6 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -227,6 +227,132 @@ def test_wildcard_ignores_reasoning_split_model_info(monkeypatch): assert _resolve_health_check_max_tokens(model_info, litellm_params) is None +# --------------------------------------------------------------------------- +# image_generation must not receive max_tokens. +# +# _update_litellm_params_for_health_check injected `max_tokens` for every +# deployment. For `mode: image_generation` that leaked into OpenAI +# `/v1/images/generations`, which strictly rejects unknown fields with +# `400 "Unknown parameter: 'max_tokens'"`, marking dall-e-* and +# gpt-image-1 as permanently unhealthy even though their actual image +# calls succeed. `messages` still gets injected (downstream +# `_filter_model_params` already strips it for non-chat handlers). +# --------------------------------------------------------------------------- + + +def test_image_generation_mode_skips_max_tokens(): + """image_generation must not receive max_tokens.""" + model_info = {"mode": "image_generation"} + litellm_params = {"model": "openai/dall-e-3", "api_key": "sk-test"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated + # connection-level params must still pass through unchanged + assert updated["api_key"] == "sk-test" + + +def test_health_check_max_tokens_value_is_ignored_for_non_chat_modes(): + """A configured `health_check_max_tokens` *value* (the int that controls + how many tokens to inject) is still skipped when the mode is outside the + allow-list — the inject decision runs before value resolution, so the + value never reaches `_resolve_health_check_max_tokens`. Note this is + distinct from `health_check_supports_max_tokens` (the bool that toggles + injection on/off per deployment).""" + model_info = {"mode": "image_generation", "health_check_max_tokens": 50} + litellm_params = {"model": "openai/dall-e-3"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated + + +def test_chat_mode_still_injects_max_tokens(): + """Regression guard: the chat-style probe payload is unchanged.""" + model_info = {"mode": "chat"} + litellm_params = {"model": "gpt-4"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +def test_no_mode_still_injects_max_tokens(): + """Regression guard: model_info without `mode` keeps the legacy path.""" + model_info: dict = {} + litellm_params = {"model": "gpt-4"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +# --------------------------------------------------------------------------- +# Allow-list behavior: only chat-style modes (chat / completion / responses) +# receive max_tokens. Every other mode is skipped by default. +# +# Per-deployment override via `health_check_supports_max_tokens` lets the +# operator force injection on (e.g. a non-listed but max_tokens-capable +# endpoint where they want to bound probe token usage) or off (e.g. a +# chat-style provider with a strict schema that rejects unknown fields). +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("mode", ["chat", "completion", "responses"]) +def test_chat_style_modes_inject_max_tokens(mode): + updated = _update_litellm_params_for_health_check( + {"mode": mode}, {"model": f"openai/dummy-{mode}"} + ) + + assert updated["max_tokens"] == 5 + + +@pytest.mark.parametrize( + "mode", + [ + "embedding", + "image_generation", + "image_edit", + "audio_speech", + "audio_transcription", + "rerank", + "video_generation", + "ocr", + "search", + "moderation", + ], +) +def test_non_chat_modes_skip_max_tokens(mode): + updated = _update_litellm_params_for_health_check( + {"mode": mode}, {"model": f"openai/dummy-{mode}"} + ) + + assert "max_tokens" not in updated + + +def test_explicit_override_true_forces_injection_outside_allowlist(): + """Operator opts a non-listed deployment in to bound probe token usage.""" + model_info = { + "mode": "image_generation", + "health_check_supports_max_tokens": True, + } + litellm_params = {"model": "openai/some-future-image-model"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert updated["max_tokens"] == 5 + + +def test_explicit_override_false_suppresses_injection_inside_allowlist(): + """Operator opts a chat-style deployment out (strict-schema provider).""" + model_info = {"mode": "chat", "health_check_supports_max_tokens": False} + litellm_params = {"model": "openai/strict-schema-chat"} + + updated = _update_litellm_params_for_health_check(model_info, litellm_params) + + assert "max_tokens" not in updated + + def test_update_litellm_params_health_check_reasoning_effort(): """model_info.health_check_reasoning_effort sets reasoning_effort for chat-style health checks.""" model_info = {"health_check_reasoning_effort": "low"}