From f8619e2000e66dba5c98a09ce0ef517a97f2ff1b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 10 Feb 2026 15:17:01 -0800 Subject: [PATCH] [Stability] Investigate + fix issue where model cost map became poorly formatted (#20895) * init: GetModelCostMap * fix * docs * docs fix * docs fixes * docs fix * test model cost map resilience * MODEL_COST_MAP_MIN_MODEL_COUNT * validate_model_cost_map * test_should_have_minimum_models_in_backup * docs fix * docs fix * fix * dos fix * docs fix * docs fix * docs fix * docs fix * validate_model_cost_map * fix * cleanup --- .../blog/model_cost_map_incident/index.md | 95 ++++++ docs/my-website/sidebars.js | 11 + litellm/constants.py | 8 + .../litellm_core_utils/get_model_cost_map.py | 201 ++++++++++-- .../test_model_cost_map_resilience.py | 291 ++++++++++++++++++ 5 files changed, 579 insertions(+), 27 deletions(-) create mode 100644 docs/my-website/blog/model_cost_map_incident/index.md create mode 100644 tests/llm_translation/test_model_cost_map_resilience.py diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md new file mode 100644 index 0000000000..7f1324e5fa --- /dev/null +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -0,0 +1,95 @@ +--- +slug: model-cost-map-incident +title: "Incident Report: Invalid model cost map on main" +date: 2026-02-10T10:00:00 +authors: + - name: Ishaan Jaffer + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/ishaanjaffer/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [incident-report, stability] +hide_table_of_contents: false +--- + +**Date:** January 27, 2026 +**Duration:** ~20 minutes +**Severity:** Low +**Status:** Resolved + +## Summary + +A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked. + +- **LLM calls and proxy routing:** No impact. +- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted. + +{/* truncate */} + +--- + +## Background + +The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call. + +```mermaid +flowchart TD + A["1. litellm.completion() receives request + litellm/main.py"] --> B["2. Route to provider + litellm/litellm_core_utils/get_llm_provider_logic.py"] + B --> C["3. LLM returns response + litellm/main.py"] + C --> D["4. Post-call: look up model in cost map + litellm/cost_calculator.py"] + D -->|"found"| E["5a. Attach cost to response"] + D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"] + E --> G["6. Return response to caller"] + F --> G + + style D fill:#fff3cd,stroke:#ffc107 + style F fill:#fff3cd,stroke:#ffc107 + style E fill:#d4edda,stroke:#28a745 + style G fill:#d4edda,stroke:#28a745 +``` + +Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request. + +--- + +## Root cause + +LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged. + +A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models. + +**Timeline:** + +1. Malformed JSON merged to `main` +2. LiteLLM installations fall back to local backup on next import +3. Users report `"This model isn't mapped yet"` for newer models +4. Bad commit identified and reverted (~20 minutes) + +--- + +## Remediation + +| # | Action | Status | Code | +|---|---|---|---| +| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [PR #20605](https://github.com/BerriAI/litellm/pull/20605) | +| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | +| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py) | +| 4 | Resilience test suite (bad hosted map, bad backup, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | +| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py) | + +Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely. + +--- + +## Other dependencies on external resources + +| Dependency | Impact if unavailable | Fallback | +|---|---|---| +| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) | +| JWT public keys (IDP/SSO) | Auth fails | None | +| OIDC UserInfo (IDP/SSO) | Auth fails | None | +| HuggingFace model API | HF provider calls fail | None | +| Ollama tags (localhost) | Ollama model list stale | Static list | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 2c3dfb2b86..28e1724ef8 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -1085,6 +1085,17 @@ const sidebars = { "troubleshoot/max_callbacks", ], }, + { + type: "category", + label: "Blog", + items: [ + { + type: "link", + label: "Incident: Broken Model Cost Map", + href: "/blog/model-cost-map-incident", + }, + ], + }, ], }; diff --git a/litellm/constants.py b/litellm/constants.py index 9c25cf7790..180315ace0 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -48,6 +48,14 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int( os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) ) DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) + +# Model cost map validation constants +MODEL_COST_MAP_MIN_MODEL_COUNT = int( + os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50) +) # Minimum number of models a fetched cost map must contain to be considered valid +MODEL_COST_MAP_MAX_SHRINK_RATIO = float( + os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5) +) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup) DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) # Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 9b86f4ca2f..e622a31745 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -8,40 +8,187 @@ export LITELLM_LOCAL_MODEL_COST_MAP=True ``` """ +import json import os +from importlib.resources import files import httpx +from litellm import verbose_logger +from litellm.constants import ( + MODEL_COST_MAP_MAX_SHRINK_RATIO, + MODEL_COST_MAP_MIN_MODEL_COUNT, +) + + +class GetModelCostMap: + """ + Handles fetching, validating, and loading the model cost map. + + Only the backup model *count* is cached (a single int). The full + backup dict is never held in memory — it is only parsed when it + needs to be *returned* as a fallback. + """ + + _backup_model_count: int = -1 # -1 = not yet loaded + + @staticmethod + def load_local_model_cost_map() -> dict: + """Load the local backup model cost map bundled with the package.""" + content = json.loads( + files("litellm") + .joinpath("model_prices_and_context_window_backup.json") + .read_text(encoding="utf-8") + ) + return content + + @classmethod + def _get_backup_model_count(cls) -> int: + """Return the number of models in the local backup (cached int).""" + if cls._backup_model_count < 0: + backup = cls.load_local_model_cost_map() + cls._backup_model_count = len(backup) + return cls._backup_model_count + + @staticmethod + def _check_is_valid_dict(fetched_map: dict) -> bool: + """Check 1: fetched map is a non-empty dict.""" + if not isinstance(fetched_map, dict): + verbose_logger.warning( + "LiteLLM: Fetched model cost map is not a dict (type=%s). " + "Falling back to local backup.", + type(fetched_map).__name__, + ) + return False + + if len(fetched_map) == 0: + verbose_logger.warning( + "LiteLLM: Fetched model cost map is empty. " + "Falling back to local backup.", + ) + return False + + return True + + @classmethod + def _check_model_count_not_reduced( + cls, + fetched_map: dict, + backup_model_count: int, + min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, + max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, + ) -> bool: + """Check 2: model count has not reduced significantly vs backup.""" + fetched_count = len(fetched_map) + + if fetched_count < min_model_count: + verbose_logger.warning( + "LiteLLM: Fetched model cost map has only %d models (minimum=%d). " + "This may indicate a corrupted upstream file. " + "Falling back to local backup.", + fetched_count, + min_model_count, + ) + return False + + if backup_model_count > 0 and fetched_count < backup_model_count * max_shrink_ratio: + verbose_logger.warning( + "LiteLLM: Fetched model cost map shrank significantly " + "(fetched=%d, backup=%d, threshold=%.0f%%). " + "This may indicate a corrupted upstream file. " + "Falling back to local backup.", + fetched_count, + backup_model_count, + max_shrink_ratio * 100, + ) + return False + + return True + + @classmethod + def validate_model_cost_map( + cls, + fetched_map: dict, + backup_model_count: int, + min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT, + max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO, + ) -> bool: + """ + Validate the integrity of a fetched model cost map. + + Runs each check in order and returns False on the first failure. + + Checks: + 1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict. + 2. ``_check_model_count_not_reduced`` -- model count meets minimum + and has not shrunk >``max_shrink_ratio`` vs backup. + + Returns True if all checks pass, False otherwise. + """ + if not cls._check_is_valid_dict(fetched_map): + return False + + if not cls._check_model_count_not_reduced( + fetched_map=fetched_map, + backup_model_count=backup_model_count, + min_model_count=min_model_count, + max_shrink_ratio=max_shrink_ratio, + ): + return False + + return True + + @staticmethod + def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict: + """ + Fetch the model cost map from a remote URL. + + Returns the parsed JSON dict. Raises on network/parse errors + (caller is expected to handle). + """ + response = httpx.get(url, timeout=timeout) + response.raise_for_status() + return response.json() + def get_model_cost_map(url: str) -> dict: - if ( - os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) - or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", False) == "True" - ): - from importlib.resources import files - import json + """ + Public entry point — returns the model cost map dict. - content = json.loads( - files("litellm") - .joinpath("model_prices_and_context_window_backup.json") - .read_text(encoding="utf-8") - ) - return content + 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. + 2. Otherwise fetches from ``url``, validates integrity, and falls back + to the local backup on any failure. + + Only the backup model count is cached (a single int) for validation. + The full backup dict is only parsed when it must be *returned* as a + fallback — it is never held in memory long-term. + """ + # Note: can't use get_secret_bool here — this runs during litellm.__init__ + # before litellm._key_management_settings is set. + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + return GetModelCostMap.load_local_model_cost_map() try: - response = httpx.get( - url, timeout=5 - ) # set a 5 second timeout for the get request - response.raise_for_status() # Raise an exception if the request is unsuccessful - content = response.json() - return content - except Exception: - from importlib.resources import files - import json - - content = json.loads( - files("litellm") - .joinpath("model_prices_and_context_window_backup.json") - .read_text(encoding="utf-8") + content = GetModelCostMap.fetch_remote_model_cost_map(url) + except Exception as e: + verbose_logger.warning( + "LiteLLM: Failed to fetch remote model cost map from %s: %s. " + "Falling back to local backup.", + url, + str(e), ) - return content + return GetModelCostMap.load_local_model_cost_map() + + # Validate using cached count (cheap int comparison, no file I/O) + if not GetModelCostMap.validate_model_cost_map( + fetched_map=content, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. " + "Using local backup instead. url=%s", + url, + ) + return GetModelCostMap.load_local_model_cost_map() + + return content diff --git a/tests/llm_translation/test_model_cost_map_resilience.py b/tests/llm_translation/test_model_cost_map_resilience.py new file mode 100644 index 0000000000..61e375eabe --- /dev/null +++ b/tests/llm_translation/test_model_cost_map_resilience.py @@ -0,0 +1,291 @@ +""" +Tests for model cost map resilience. + +Simulates: +- A bad (invalid JSON) model cost map upstream +- A bad (empty/missing) backup model cost map +- Verifies litellm.completion() still works even with a broken cost map +- Verifies litellm.get_model_info() raises the expected error for unmapped models +- Verifies the integrity validation helper catches corrupted maps +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +) + +import litellm +from litellm.litellm_core_utils.get_model_cost_map import ( + GetModelCostMap, + get_model_cost_map, +) + + +class TestCheckIsValidDict: + """Unit tests for _check_is_valid_dict.""" + + def test_should_reject_non_dict(self): + """Non-dict should fail.""" + assert GetModelCostMap._check_is_valid_dict("not a dict") is False + + def test_should_reject_empty_dict(self): + """Empty dict should fail.""" + assert GetModelCostMap._check_is_valid_dict({}) is False + + def test_should_reject_list(self): + """List should fail.""" + assert GetModelCostMap._check_is_valid_dict([1, 2, 3]) is False + + def test_should_reject_none(self): + """None should fail.""" + assert GetModelCostMap._check_is_valid_dict(None) is False + + def test_should_accept_non_empty_dict(self): + """Non-empty dict should pass.""" + assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True + + +class TestCheckModelCountNotReduced: + """Unit tests for _check_model_count_not_reduced.""" + + def test_should_reject_too_few_models(self): + """Fetched map with fewer models than min_model_count should fail.""" + small_map = {f"model-{i}": {} for i in range(5)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=small_map, backup_model_count=0, min_model_count=10 + ) + is False + ) + + def test_should_reject_significant_shrinkage(self): + """Fetched map that shrunk >50% vs backup should fail.""" + fetched = {f"model-{i}": {} for i in range(40)} # 40% of 100 + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is False + ) + + def test_should_accept_when_above_threshold(self): + """Fetched map at 60% of backup (above 50% threshold) should pass.""" + fetched = {f"model-{i}": {} for i in range(60)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_growth(self): + """Fetched map larger than backup should pass.""" + fetched = {f"model-{i}": {} for i in range(120)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_with_empty_backup(self): + """When backup is empty, only min_model_count matters.""" + fetched = {f"model-{i}": {} for i in range(15)} + assert ( + GetModelCostMap._check_model_count_not_reduced( + fetched_map=fetched, backup_model_count=0, min_model_count=10 + ) + is True + ) + + +class TestValidateModelCostMap: + """Unit tests for validate_model_cost_map (combines both checks).""" + + def test_should_reject_non_dict(self): + """Non-dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map="not a dict", backup_model_count=0) is False + + def test_should_reject_empty_map(self): + """Empty dict should fail at check 1.""" + assert GetModelCostMap.validate_model_cost_map(fetched_map={}, backup_model_count=0) is False + + def test_should_reject_significant_shrinkage(self): + """Should fail at check 2 (shrinkage).""" + fetched = {f"model-{i}": {} for i in range(40)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is False + ) + + def test_should_accept_valid_map(self): + """Should pass both checks.""" + fetched = {f"model-{i}": {} for i in range(120)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + def test_should_accept_equal_size_map(self): + """Equal size should pass both checks.""" + fetched = {f"model-{i}": {} for i in range(100)} + assert ( + GetModelCostMap.validate_model_cost_map( + fetched_map=fetched, backup_model_count=100, min_model_count=10 + ) + is True + ) + + +class TestGetModelCostMapFallback: + """Tests for get_model_cost_map fallback behavior with bad upstream.""" + + def test_should_fallback_to_backup_on_invalid_json(self): + """When upstream returns invalid JSON, should fall back to local backup.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Should have fallen back to backup — backup always has models + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_to_backup_on_network_error(self): + """When upstream is unreachable, should fall back to local backup.""" + with patch("httpx.get", side_effect=Exception("Connection refused")): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_when_fetched_map_is_empty(self): + """When upstream returns valid JSON but empty dict, should fall back.""" + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = {} # empty map + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Should have fallen back to backup since empty map fails validation + assert isinstance(result, dict) + assert len(result) > 0 + + def test_should_fallback_when_fetched_map_shrinks_dramatically(self): + """When upstream returns far fewer models than backup, should fall back.""" + tiny_map = {f"model-{i}": {"litellm_provider": "test"} for i in range(11)} + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = tiny_map + + with patch("httpx.get", return_value=mock_response): + result = get_model_cost_map("https://fake-url.com/model_prices.json") + + # Backup has thousands of models; 11 is a massive shrinkage → fallback + assert len(result) > 11 + + def test_should_use_local_map_when_env_var_set(self): + """LITELLM_LOCAL_MODEL_COST_MAP=True should skip remote fetch entirely.""" + with patch.dict(os.environ, {"LITELLM_LOCAL_MODEL_COST_MAP": "True"}): + with patch("httpx.get") as mock_get: + result = get_model_cost_map( + "https://fake-url.com/model_prices.json" + ) + mock_get.assert_not_called() + + assert isinstance(result, dict) + assert len(result) > 0 + + +class TestBackupModelCostMapExists: + """Validates the local backup file is always present and valid.""" + + def test_should_have_backup_file(self): + """The backup model cost map must exist and be loadable.""" + backup = GetModelCostMap.load_local_model_cost_map() + assert isinstance(backup, dict) + assert len(backup) > 0, "Backup model cost map is empty" + + def test_should_have_minimum_models_in_backup(self): + """The backup must contain a reasonable number of models.""" + backup = GetModelCostMap.load_local_model_cost_map() + assert len(backup) > 100, ( + f"Backup has only {len(backup)} models, expected > 100" + ) + + +class TestBadHostedModelCostMap: + """ + Simulates the hosted model cost map being bad (invalid JSON / corrupted). + + When the hosted map is bad, get_model_cost_map() falls back to the local + backup. These tests verify that after fallback: + - get_model_info() still works for models in the backup + - litellm.completion() still works + """ + + def test_should_model_info_pass_after_bad_hosted_map(self): + """ + If the hosted map is bad, get_model_cost_map falls back to the local + backup. get_model_info should still work for models in the backup. + """ + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_response): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + + original = litellm.model_cost + litellm.model_cost = fallback_map + try: + # gpt-4o is in every backup — should work fine + info = litellm.get_model_info("gpt-4o") + assert info is not None + assert info["input_cost_per_token"] > 0 + finally: + litellm.model_cost = original + + def test_should_completion_pass_after_bad_hosted_map(self): + """ + If the hosted map is bad, litellm.completion() should still work. + + Uses litellm's built-in mock_response param so the real completion + path is exercised (routing, cost calculator, logging) without + needing API credentials. + """ + # Simulate bad hosted map → fallback to backup + mock_http = MagicMock() + mock_http.raise_for_status = MagicMock() + mock_http.json.side_effect = json.JSONDecodeError("bad json", "", 0) + + with patch("httpx.get", return_value=mock_http): + fallback_map = get_model_cost_map("https://fake-url.com/bad.json") + + original = litellm.model_cost + litellm.model_cost = fallback_map + try: + # mock_response goes through the real completion path — + # routing, cost calculator, logging — but skips the HTTP call + response = litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "say hi"}], + mock_response="hello from mock", + ) + assert response is not None + assert response.choices[0].message.content == "hello from mock" + finally: + litellm.model_cost = original