tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-14 11:08:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

(fix) test_tokenizers

Browse Source
This commit is contained in:
ishaan-jaff
2024-02-23 10:08:07 -08:00
parent 44b8270bf6
commit ff815e93a8
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/tests/test_token_counter.py
+1 -1
View File
@@ -53,7 +53,7 @@ def test_tokenizers():
try:
### test the openai, claude, cohere and llama2 tokenizers.
### The tokenizer value should be different for all
sample_text = "Hellö World, this is my input string!"
sample_text = "Hellö World, this is my input string! My name is ishaan CTO"
# openai tokenizer
openai_tokens = token_counter(model="gpt-3.5-turbo", text=sample_text)