mirror of
https://github.com/tiennm99/litellm.git
synced 2026-06-18 00:48:01 +00:00
fix(token-counter): fix test isolation and encode() return type normalization
Two independent fixes for test_token_counter.py failures in CI: 1. test_disable_hf_tokenizer_download leaked litellm.disable_hf_tokenizer_download=True because pytest.MonkeyPatch() was never undone. The setting persisted into the alphabetically-subsequent test_llama2/3_tokenizer_api_failure tests, causing _select_tokenizer_helper to short-circuit before calling from_pretrained. Fix: wrap the test body in try/finally and call monkeypatch.undo(). 2. encode() returns a HuggingFace Encoding object when the HF tokenizer loads, but falls back to returning a plain List[int] (tiktoken) when the model hub is unreachable. test_encoding_and_decoding called .ids on the result, which raises AttributeError when the list-based fallback is active. Fix: normalize encode() to always return List[int] by extracting .ids when present, and remove the now-unnecessary .ids access in the test. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -582,14 +582,15 @@ class TestTokenizerSelection(unittest.TestCase):
|
||||
|
||||
@patch("litellm.utils._return_huggingface_tokenizer")
|
||||
def test_disable_hf_tokenizer_download(self, mock_return_huggingface_tokenizer):
|
||||
# Use pytest.MonkeyPatch() directly instead of fixture
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", True)
|
||||
|
||||
result = _select_tokenizer_helper("grok-32r22r")
|
||||
mock_return_huggingface_tokenizer.assert_not_called()
|
||||
assert result["type"] == "openai_tokenizer"
|
||||
assert result["tokenizer"] == encoding
|
||||
try:
|
||||
result = _select_tokenizer_helper("grok-32r22r")
|
||||
mock_return_huggingface_tokenizer.assert_not_called()
|
||||
assert result["type"] == "openai_tokenizer"
|
||||
assert result["tokenizer"] == encoding
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
Reference in New Issue
Block a user