tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-02 06:22:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update tests/test_litellm/litellm_core_utils/test_token_counter.py

Browse Source 
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
jquinter
2026-02-17 19:52:06 -03:00
committed by GitHub
co-authored by greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
parent 16ca7f4f96
commit afbfbf3cae
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_litellm/litellm_core_utils/test_token_counter.py
+5 -4
View File
@@ -213,10 +213,11 @@ def test_tokenizers():
# llama2 may fall back to the tiktoken tokenizer when the HuggingFace
# model hub is unreachable (e.g. in CI). In that case the count will
# equal the openai count and the differentiation assertion is skipped.
if openai_tokens != llama2_tokens:
assert (
llama2_tokens != llama3_tokens_1
), "Token values are not different."
if openai_tokens == llama2_tokens:
pytest.skip("llama2 fell back to tiktoken (HF hub unreachable); skipping differentiation assertion")
assert (
llama2_tokens != llama3_tokens_1
), "Token values are not different."
assert (
llama3_tokens_1 == llama3_tokens_2