fix token counter bug

2026-08-09 02:24:40 +00:00 · 2024-06-05 23:40:55 -07:00
parent 2e95758ada
commit f4c49755a0
1 changed files with 1 additions and 0 deletions
@@ -4060,6 +4060,7 @@ def openai_token_counter(
                    for c in value:
                        if c["type"] == "text":
                            text += c["text"]
+                            num_tokens += len(encoding.encode(c["text"], disallowed_special=()))
                        elif c["type"] == "image_url":
                            if isinstance(c["image_url"], dict):
                                image_url_dict = c["image_url"]