From df57e9247acb202dfe19c0bbcacd664136dad22a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 3 Nov 2023 17:59:34 -0700 Subject: [PATCH] (fix) hf calculating usage non blocking --- litellm/llms/huggingface_restapi.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index f4719c53c0..62787a0b5f 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -333,15 +333,25 @@ def completion( "content" ] = completion_response[0]["generated_text"] ## CALCULATING USAGE - prompt_tokens = len( - encoding.encode(input_text) - ) ##[TODO] use the llama2 tokenizer here + prompt_tokens = 0 + try: + prompt_tokens = len( + encoding.encode(input_text) + ) ##[TODO] use the llama2 tokenizer here + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass print_verbose(f'output: {model_response["choices"][0]["message"]}') output_text = model_response["choices"][0]["message"].get("content", "") if output_text is not None and len(output_text) > 0: - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) ##[TODO] use the llama2 tokenizer here + completion_tokens = 0 + try: + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content", "")) + ) ##[TODO] use the llama2 tokenizer here + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass else: completion_tokens = 0