tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-04 01:06:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix token_counter with special token input

Browse Source
This commit is contained in:
Yikai Zhao
2025-08-07 22:58:07 +08:00
parent dfada882f1
commit 4fdeff8e1a
2 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/litellm_core_utils/token_counter.py
+1 -1
View File
@@ -529,7 +529,7 @@ def _get_count_function(
encoding = tiktoken.get_encoding("cl100k_base")
def count_tokens(text: str) -> int:
return len(encoding.encode(text))
return len(encoding.encode(text, disallowed_special=()))
else:
raise ValueError("Unsupported tokenizer type")
tests/test_litellm/litellm_core_utils/test_token_counter.py
+1
View File
@@ -451,6 +451,7 @@ def test_img_url_token_counter(img_url):
def test_token_encode_disallowed_special():
encode(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>")
token_counter(model="gpt-3.5-turbo", text="Hello, world! <|endoftext|>")
def test_token_counter():