From 6304760dd0e2436c9e8e10c5f382d8daa0d3c7e3 Mon Sep 17 00:00:00 2001 From: Rahul Kataria Date: Sun, 12 May 2024 15:04:45 +0530 Subject: [PATCH 1/4] [Optimize] Optimize the code in caching file --- litellm/caching.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 83cfe060b1..258d7b18b2 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -374,10 +374,11 @@ class RedisCache(BaseCache): f"Set ASYNC Redis Cache PIPELINE: key: {cache_key}\nValue {cache_value}\nttl={ttl}" ) # Set the value with a TTL if it's provided. + json_cache_value = json.dumps(cache_value) if ttl is not None: - pipe.setex(cache_key, ttl, json.dumps(cache_value)) + pipe.setex(cache_key, ttl, json_cache_value) else: - pipe.set(cache_key, json.dumps(cache_value)) + pipe.set(cache_key, json_cache_value) # Execute the pipeline and return the results. results = await pipe.execute() @@ -810,9 +811,7 @@ class RedisSemanticCache(BaseCache): # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # create an embedding for prompt embedding_response = litellm.embedding( @@ -847,9 +846,7 @@ class RedisSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # convert to embedding embedding_response = litellm.embedding( @@ -909,9 +906,7 @@ class RedisSemanticCache(BaseCache): # get the prompt messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) # create an embedding for prompt router_model_names = ( [m["model_name"] for m in llm_model_list] @@ -964,9 +959,7 @@ class RedisSemanticCache(BaseCache): # get the messages messages = kwargs["messages"] - prompt = "" - for message in messages: - prompt += message["content"] + prompt = "".join(message["content"] for message in messages) router_model_names = ( [m["model_name"] for m in llm_model_list] From 3f146b2c7e2c094458342ba52b3d0943f2df8be4 Mon Sep 17 00:00:00 2001 From: Rahul Kataria Date: Sun, 12 May 2024 16:37:50 +0530 Subject: [PATCH 2/4] [Reset Code] --- litellm/caching.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 258d7b18b2..466f4d1863 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -374,11 +374,10 @@ class RedisCache(BaseCache): f"Set ASYNC Redis Cache PIPELINE: key: {cache_key}\nValue {cache_value}\nttl={ttl}" ) # Set the value with a TTL if it's provided. - json_cache_value = json.dumps(cache_value) if ttl is not None: - pipe.setex(cache_key, ttl, json_cache_value) + pipe.setex(cache_key, ttl, json.dumps(cache_value)) else: - pipe.set(cache_key, json_cache_value) + pipe.set(cache_key, json.dumps(cache_value)) # Execute the pipeline and return the results. results = await pipe.execute() From 689221b5c72c2a1728ae0710fe1d27f4e6c7434f Mon Sep 17 00:00:00 2001 From: Rahul Kataria Date: Sun, 12 May 2024 17:04:18 +0530 Subject: [PATCH 3/4] [Optimize] Optimize code in caching file --- litellm/caching.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 466f4d1863..ccb62b8827 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -373,11 +373,12 @@ class RedisCache(BaseCache): print_verbose( f"Set ASYNC Redis Cache PIPELINE: key: {cache_key}\nValue {cache_value}\nttl={ttl}" ) + json_cache_value = json.dumps(cache_value) # Set the value with a TTL if it's provided. if ttl is not None: - pipe.setex(cache_key, ttl, json.dumps(cache_value)) + pipe.setex(cache_key, ttl, json_cache_value) else: - pipe.set(cache_key, json.dumps(cache_value)) + pipe.set(cache_key, json_cache_value) # Execute the pipeline and return the results. results = await pipe.execute() From d57ecf337194301c9164e38225eb7bbd58595616 Mon Sep 17 00:00:00 2001 From: Rahul Kataria Date: Sun, 12 May 2024 18:05:57 +0530 Subject: [PATCH 4/4] Remove duplicate code in router_strategy --- litellm/router_strategy/lowest_cost.py | 3 --- litellm/router_strategy/lowest_latency.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index fde7781b9b..1670490e16 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -101,9 +101,6 @@ class LowestCostLoggingHandler(CustomLogger): if precise_minute not in request_count_dict[id]: request_count_dict[id][precise_minute] = {} - if precise_minute not in request_count_dict[id]: - request_count_dict[id][precise_minute] = {} - ## TPM request_count_dict[id][precise_minute]["tpm"] = ( request_count_dict[id][precise_minute].get("tpm", 0) + total_tokens diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index a7b93d344d..81a0133a90 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -115,9 +115,6 @@ class LowestLatencyLoggingHandler(CustomLogger): if precise_minute not in request_count_dict[id]: request_count_dict[id][precise_minute] = {} - if precise_minute not in request_count_dict[id]: - request_count_dict[id][precise_minute] = {} - ## TPM request_count_dict[id][precise_minute]["tpm"] = ( request_count_dict[id][precise_minute].get("tpm", 0) + total_tokens