From 128cf4a81d8eca2ba8d3c8c7e1fbe8fd1d8a1fd3 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 22 Jan 2024 15:05:09 -0800
Subject: [PATCH] fix(utils.py): move from pkg_resources to importlib

---
 litellm/utils.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index 1fea89996e..699d63960a 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2492,24 +2492,20 @@ def get_replicate_completion_pricing(completion_response=None, total_time=0.0):
 
 
 def _select_tokenizer(model: str):
-    # cohere
-    import pkg_resources
+    from importlib import resources
 
     if model in litellm.cohere_models:
+        # cohere
         tokenizer = Tokenizer.from_pretrained("Cohere/command-nightly")
         return {"type": "huggingface_tokenizer", "tokenizer": tokenizer}
     # anthropic
     elif model in litellm.anthropic_models:
-        # Read the JSON file
-        filename = pkg_resources.resource_filename(
-            __name__, "llms/tokenizers/anthropic_tokenizer.json"
-        )
-        with open(filename, "r") as f:
+        with resources.open_text(
+            "litellm.llms.tokenizers", "anthropic_tokenizer.json"
+        ) as f:
             json_data = json.load(f)
-        # Decode the JSON data from utf-8
-        json_data_decoded = json.dumps(json_data, ensure_ascii=False)
-        # Convert to str
-        json_str = str(json_data_decoded)
+        # Convert to str (if necessary)
+        json_str = json.dumps(json_data)
         # load tokenizer
         tokenizer = Tokenizer.from_str(json_str)
         return {"type": "huggingface_tokenizer", "tokenizer": tokenizer}