From c1a81d90a60951ad583446db1d6226f8cb814be7 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Wed, 24 Apr 2024 15:15:19 +0200 Subject: [PATCH 1/9] build(caching.py) - add disk cache object --- litellm/caching.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++ poetry.lock | 13 +++++++- pyproject.toml | 1 + 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/litellm/caching.py b/litellm/caching.py index 83cfe060b1..b9fc6b28f0 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -15,6 +15,7 @@ from openai._models import BaseModel as OpenAIObject from litellm._logging import verbose_logger from litellm.types.services import ServiceLoggerPayload, ServiceTypes import traceback +import diskcache as dc def print_verbose(print_statement): @@ -1914,6 +1915,79 @@ class Cache: await self.cache.disconnect() +class DiskCache(BaseCache): + def __init__(self, cache_dir: str = ".cache"): + # if users don't provider one, use the default litellm cache + self.disk_cache = dc.Cache(cache_dir) + + def set_cache(self, key, value, **kwargs): + print_verbose("DiskCache: set_cache") + if "ttl" in kwargs: + self.disk_cache.set(key, value, expire=kwargs["ttl"]) + else: + self.disk_cache.set(key, value) + + async def async_set_cache(self, key, value, **kwargs): + self.set_cache(key=key, value=value, **kwargs) + + async def async_set_cache_pipeline(self, cache_list, ttl=None): + for cache_key, cache_value in cache_list: + if ttl is not None: + self.set_cache(key=cache_key, value=cache_value, ttl=ttl) + else: + self.set_cache(key=cache_key, value=cache_value) + + def get_cache(self, key, **kwargs): + original_cached_response = self.disk_cache.get(key) + if original_cached_response: + try: + cached_response = json.loads(original_cached_response) + except: + cached_response = original_cached_response + return cached_response + return None + + def batch_get_cache(self, keys: list, **kwargs): + return_val = [] + for k in keys: + val = self.get_cache(key=k, **kwargs) + return_val.append(val) + return return_val + + def increment_cache(self, key, value: int, **kwargs) -> int: + # get the value + init_value = self.get_cache(key=key) or 0 + value = init_value + value + self.set_cache(key, value, **kwargs) + return value + + async def async_get_cache(self, key, **kwargs): + return self.get_cache(key=key, **kwargs) + + async def async_batch_get_cache(self, keys: list, **kwargs): + return_val = [] + for k in keys: + val = self.get_cache(key=k, **kwargs) + return_val.append(val) + return return_val + + async def async_increment(self, key, value: int, **kwargs) -> int: + # get the value + init_value = await self.async_get_cache(key=key) or 0 + value = init_value + value + await self.async_set_cache(key, value, **kwargs) + return value + + def flush_cache(self): + self.disk_cache.clear() + + async def disconnect(self): + pass + + def delete_cache(self, key): + self.disk_cache.pop(key) + + def enable_cache( type: Optional[Literal["local", "redis", "s3"]] = "local", host: Optional[str] = None, diff --git a/poetry.lock b/poetry.lock index f3f7d40d5b..842dfe3108 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,6 +605,17 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] +[[package]] +name = "diskcache" +version = "5.6.3" +description = "Disk Cache -- Disk and file backed persistent cache." +optional = true +python-versions = ">=3" +files = [ + {file = "diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19"}, + {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"}, +] + [[package]] name = "distro" version = "1.9.0" @@ -2674,4 +2685,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "cryptography", "fastapi", "fastapi- [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "ff38be297294f084a739ef869d41d3d80f09c80e1d05d2963073d49790f33f37" +content-hash = "9d2c78a508742a83a9dce1f3f55f8ef9cac8a453a863eaa91a179f031894be1b" diff --git a/pyproject.toml b/pyproject.toml index a9854cf692..81e2e9abff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ azure-identity = {version = "^1.15.0", optional = true} azure-keyvault-secrets = {version = "^4.8.0", optional = true} google-cloud-kms = {version = "^2.21.3", optional = true} resend = {version = "^0.8.0", optional = true} +diskcache = {version = "^5.6.3", optional = true} [tool.poetry.extras] proxy = [ From ac27f431a4396526ccce3586c0b23b4277873e85 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Wed, 24 Apr 2024 15:59:22 +0200 Subject: [PATCH 2/9] test(test_caching.py): add disk cache test when using completion --- litellm/tests/test_caching.py | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 903ce69c77..42b175c72f 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -875,6 +875,66 @@ async def test_redis_cache_acompletion_stream_bedrock(): print(e) raise e +def test_disk_cache_completion(): + litellm.set_verbose = False + + random_number = random.randint( + 1, 100000 + ) # add a random number to ensure it's always adding / reading from cache + messages = [ + {"role": "user", "content": f"write a one sentence poem about: {random_number}"} + ] + litellm.cache = Cache( + type="disk", + ) + print("test2 for Redis Caching - non streaming") + response1 = completion( + model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 + ) + response2 = completion( + model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 + ) + response3 = completion( + model="gpt-3.5-turbo", messages=messages, caching=True, temperature=0.5 + ) + + print("\nresponse 1", response1) + print("\nresponse 2", response2) + print("\nresponse 3", response3) + # print("\nresponse 4", response4) + litellm.cache = None + litellm.success_callback = [] + litellm._async_success_callback = [] + + """ + 1 & 2 should be exactly the same + 1 & 3 should be different, since input params are diff + 1 & 4 should be diff, since models are diff + """ + if ( + response1["choices"][0]["message"]["content"] + != response2["choices"][0]["message"]["content"] + ): # 1 and 2 should be the same + # 1&2 have the exact same input params. This MUST Be a CACHE HIT + print(f"response1: {response1}") + print(f"response2: {response2}") + pytest.fail(f"Error occurred:") + if ( + response1["choices"][0]["message"]["content"] + == response3["choices"][0]["message"]["content"] + ): + # if input params like seed, max_tokens are diff it should NOT be a cache hit + print(f"response1: {response1}") + print(f"response3: {response3}") + pytest.fail( + f"Response 1 == response 3. Same model, diff params shoudl not cache Error" + f" occurred:" + ) + + assert response1.id == response2.id + assert response1.created == response2.created + assert response1.choices[0].message.content == response2.choices[0].message.content + @pytest.mark.skip(reason="AWS Suspended Account") @pytest.mark.asyncio From 004877c7e5e7cbee43ea2095620c80249effc512 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 10:00:25 +0200 Subject: [PATCH 3/9] build(caching.py): add disk option for cache --- litellm/caching.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index b9fc6b28f0..63e2cd39a6 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1449,7 +1449,7 @@ class DualCache(BaseCache): class Cache: def __init__( self, - type: Optional[Literal["local", "redis", "redis-semantic", "s3"]] = "local", + type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -1498,7 +1498,7 @@ class Cache: Initializes the cache based on the given type. Args: - type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", or "s3". Defaults to "local". + type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "s3" or "disk". Defaults to "local". host (str, optional): The host address for the Redis cache. Required if type is "redis". port (int, optional): The port number for the Redis cache. Required if type is "redis". password (str, optional): The password for the Redis cache. Required if type is "redis". @@ -1544,6 +1544,8 @@ class Cache: s3_path=s3_path, **kwargs, ) + elif type == "disk": + self.cache = DiskCache() if "cache" not in litellm.input_callback: litellm.input_callback.append("cache") if "cache" not in litellm.success_callback: @@ -1916,7 +1918,7 @@ class Cache: class DiskCache(BaseCache): - def __init__(self, cache_dir: str = ".cache"): + def __init__(self, cache_dir: str = ".litellm_cache"): # if users don't provider one, use the default litellm cache self.disk_cache = dc.Cache(cache_dir) @@ -1989,7 +1991,7 @@ class DiskCache(BaseCache): def enable_cache( - type: Optional[Literal["local", "redis", "s3"]] = "local", + type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -2018,7 +2020,7 @@ def enable_cache( Enable cache with the specified configuration. Args: - type (Optional[Literal["local", "redis"]]): The type of cache to enable. Defaults to "local". + type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local". host (Optional[str]): The host address of the cache server. Defaults to None. port (Optional[str]): The port number of the cache server. Defaults to None. password (Optional[str]): The password for the cache server. Defaults to None. @@ -2054,7 +2056,7 @@ def enable_cache( def update_cache( - type: Optional[Literal["local", "redis"]] = "local", + type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, @@ -2083,7 +2085,7 @@ def update_cache( Update the cache for LiteLLM. Args: - type (Optional[Literal["local", "redis"]]): The type of cache. Defaults to "local". + type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local". host (Optional[str]): The host of the cache. Defaults to None. port (Optional[str]): The port of the cache. Defaults to None. password (Optional[str]): The password for the cache. Defaults to None. From 9b2dcb2807dbf6ead895ce1cc09f71e9c4fae98f Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 10:56:36 +0200 Subject: [PATCH 4/9] chore(pyproject.toml): add diskcache as extra --- poetry.lock | 3 ++- pyproject.toml | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 842dfe3108..0c1b26a9ae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2679,10 +2679,11 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] +extra-cache = ["diskcache"] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-kms", "prisma", "resend"] proxy = ["PyJWT", "apscheduler", "backoff", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "orjson", "python-multipart", "pyyaml", "rq", "uvicorn"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "9d2c78a508742a83a9dce1f3f55f8ef9cac8a453a863eaa91a179f031894be1b" +content-hash = "a21a41c1f683d7cb9a07fa76ee396b7da95cebf8c82d44de92e318916436a506" diff --git a/pyproject.toml b/pyproject.toml index 81e2e9abff..486a04f3c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,9 @@ extra_proxy = [ "resend" ] +extra_cache = [ + "diskcache" +] [tool.poetry.scripts] litellm = 'litellm:run_server' From 7ee07cd961fe7913f110cc3b7f46818eb48bd3c8 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 10:57:24 +0200 Subject: [PATCH 5/9] test(test_caching.py): use mock_response in disk cache test --- litellm/tests/test_caching.py | 40 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/litellm/tests/test_caching.py b/litellm/tests/test_caching.py index 42b175c72f..2f0f1dbfe6 100644 --- a/litellm/tests/test_caching.py +++ b/litellm/tests/test_caching.py @@ -599,7 +599,10 @@ def test_redis_cache_completion(): ) print("test2 for Redis Caching - non streaming") response1 = completion( - model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 + model="gpt-3.5-turbo", + messages=messages, + caching=True, + max_tokens=20, ) response2 = completion( model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 @@ -653,7 +656,6 @@ def test_redis_cache_completion(): assert response1.created == response2.created assert response1.choices[0].message.content == response2.choices[0].message.content - # test_redis_cache_completion() @@ -887,15 +889,32 @@ def test_disk_cache_completion(): litellm.cache = Cache( type="disk", ) - print("test2 for Redis Caching - non streaming") + response1 = completion( - model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 + model="gpt-3.5-turbo", + messages=messages, + caching=True, + max_tokens=20, + mock_response="This number is so great!", ) + # response2 is mocked to a different response from response1, + # but the completion from the cache should be used instead of the mock + # response since the input is the same as response1 response2 = completion( - model="gpt-3.5-turbo", messages=messages, caching=True, max_tokens=20 + model="gpt-3.5-turbo", + messages=messages, + caching=True, + max_tokens=20, + mock_response="This number is awful!", ) + # Since the parameters are not the same as response1, response3 should actually + # be the mock response response3 = completion( - model="gpt-3.5-turbo", messages=messages, caching=True, temperature=0.5 + model="gpt-3.5-turbo", + messages=messages, + caching=True, + temperature=0.5, + mock_response="This number is awful!", ) print("\nresponse 1", response1) @@ -906,11 +925,8 @@ def test_disk_cache_completion(): litellm.success_callback = [] litellm._async_success_callback = [] - """ - 1 & 2 should be exactly the same - 1 & 3 should be different, since input params are diff - 1 & 4 should be diff, since models are diff - """ + # 1 & 2 should be exactly the same + # 1 & 3 should be different, since input params are diff if ( response1["choices"][0]["message"]["content"] != response2["choices"][0]["message"]["content"] @@ -923,7 +939,7 @@ def test_disk_cache_completion(): response1["choices"][0]["message"]["content"] == response3["choices"][0]["message"]["content"] ): - # if input params like seed, max_tokens are diff it should NOT be a cache hit + # if input params like max_tokens, temperature are diff it should NOT be a cache hit print(f"response1: {response1}") print(f"response3: {response3}") pytest.fail( From c1ba4ec0780c47c950bf4baacdc0c880aad91b46 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 11:19:14 +0200 Subject: [PATCH 6/9] chore: add diskcache as extra dependency --- litellm/caching.py | 2 +- poetry.lock | 4 ++-- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 63e2cd39a6..6906a1a98a 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -15,7 +15,6 @@ from openai._models import BaseModel as OpenAIObject from litellm._logging import verbose_logger from litellm.types.services import ServiceLoggerPayload, ServiceTypes import traceback -import diskcache as dc def print_verbose(print_statement): @@ -1919,6 +1918,7 @@ class Cache: class DiskCache(BaseCache): def __init__(self, cache_dir: str = ".litellm_cache"): + import diskcache as dc # if users don't provider one, use the default litellm cache self.disk_cache = dc.Cache(cache_dir) diff --git a/poetry.lock b/poetry.lock index 0c1b26a9ae..2911806cb3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2679,11 +2679,11 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] -extra-cache = ["diskcache"] +disk-caching = ["diskcache"] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-kms", "prisma", "resend"] proxy = ["PyJWT", "apscheduler", "backoff", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "orjson", "python-multipart", "pyyaml", "rq", "uvicorn"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "a21a41c1f683d7cb9a07fa76ee396b7da95cebf8c82d44de92e318916436a506" +content-hash = "aa6283846b24f5703626c2de10459254166551f469c34e75b229adafb6985eba" diff --git a/pyproject.toml b/pyproject.toml index 486a04f3c7..ca01141d2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ extra_proxy = [ "resend" ] -extra_cache = [ +disk_caching = [ "diskcache" ] From 79c3d39d670d3e7d2012bafc05d9dc8cec44eea0 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 12:04:54 +0200 Subject: [PATCH 7/9] build(caching.py): move diskcache import inside class and add cache_dir argument to Cache --- litellm/caching.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/litellm/caching.py b/litellm/caching.py index 6906a1a98a..8aec7efdc9 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -1491,6 +1491,7 @@ class Cache: redis_semantic_cache_use_async=False, redis_semantic_cache_embedding_model="text-embedding-ada-002", redis_flush_size=None, + disk_cache_dir=None, **kwargs, ): """ @@ -1544,7 +1545,7 @@ class Cache: **kwargs, ) elif type == "disk": - self.cache = DiskCache() + self.cache = DiskCache(disk_cache_dir=disk_cache_dir) if "cache" not in litellm.input_callback: litellm.input_callback.append("cache") if "cache" not in litellm.success_callback: @@ -1917,10 +1918,14 @@ class Cache: class DiskCache(BaseCache): - def __init__(self, cache_dir: str = ".litellm_cache"): + def __init__(self, disk_cache_dir: Optional[str] = None): import diskcache as dc + # if users don't provider one, use the default litellm cache - self.disk_cache = dc.Cache(cache_dir) + if disk_cache_dir is None: + self.disk_cache = dc.Cache(".litellm_cache") + else: + self.disk_cache = dc.Cache(disk_cache_dir) def set_cache(self, key, value, **kwargs): print_verbose("DiskCache: set_cache") From 9c1d312fddfc9f19470af2c359200914b33efe54 Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 12:17:03 +0200 Subject: [PATCH 8/9] docs: add disk cache doc and update cache arguments --- .../caching/{redis_cache.md => all_caches.md} | 70 ++++++++++++++----- docs/my-website/docs/caching/local_caching.md | 2 +- docs/my-website/sidebars.js | 2 +- 3 files changed, 56 insertions(+), 18 deletions(-) rename docs/my-website/docs/caching/{redis_cache.md => all_caches.md} (82%) diff --git a/docs/my-website/docs/caching/redis_cache.md b/docs/my-website/docs/caching/all_caches.md similarity index 82% rename from docs/my-website/docs/caching/redis_cache.md rename to docs/my-website/docs/caching/all_caches.md index b00a118c12..8f4196856e 100644 --- a/docs/my-website/docs/caching/redis_cache.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Caching - In-Memory, Redis, s3, Redis Semantic Cache +# Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching.py) @@ -11,7 +11,7 @@ Need to use Caching on LiteLLM Proxy Server? Doc here: [Caching Proxy Server](ht ::: -## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic Cache +## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic, Disk Cache @@ -159,7 +159,7 @@ litellm.cache = Cache() # Make completion calls response1 = completion( model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Tell me a joke."}] + messages=[{"role": "user", "content": "Tell me a joke."}], caching=True ) response2 = completion( @@ -174,6 +174,33 @@ response2 = completion( + + +### Quick Start + +```python +import litellm +from litellm import completion +from litellm.caching import Cache +litellm.cache = Cache(type="disk") + +# Make completion calls +response1 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}], + caching=True +) +response2 = completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Tell me a joke."}], + caching=True +) + +# response1 == response2, response 1 is cached + +``` + + @@ -191,13 +218,13 @@ Advanced Params ```python litellm.enable_cache( - type: Optional[Literal["local", "redis"]] = "local", + type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, supported_call_types: Optional[ - List[Literal["completion", "acompletion", "embedding", "aembedding"]] - ] = ["completion", "acompletion", "embedding", "aembedding"], + List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]] + ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"], **kwargs, ) ``` @@ -215,13 +242,13 @@ Update the Cache params ```python litellm.update_cache( - type: Optional[Literal["local", "redis"]] = "local", + type: Optional[Literal["local", "redis", "s3", "disk"]] = "local", host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, supported_call_types: Optional[ - List[Literal["completion", "acompletion", "embedding", "aembedding"]] - ] = ["completion", "acompletion", "embedding", "aembedding"], + List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]] + ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"], **kwargs, ) ``` @@ -276,22 +303,29 @@ cache.get_cache = get_cache ```python def __init__( self, - type: Optional[Literal["local", "redis", "s3"]] = "local", + type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local", supported_call_types: Optional[ - List[Literal["completion", "acompletion", "embedding", "aembedding"]] - ] = ["completion", "acompletion", "embedding", "aembedding"], # A list of litellm call types to cache for. Defaults to caching for all litellm call types. - + List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]] + ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"], + ttl: Optional[float] = None, + default_in_memory_ttl: Optional[float] = None, + # redis cache params host: Optional[str] = None, port: Optional[str] = None, password: Optional[str] = None, - + namespace: Optional[str] = None, + default_in_redis_ttl: Optional[float] = None, + similarity_threshold: Optional[float] = None, + redis_semantic_cache_use_async=False, + redis_semantic_cache_embedding_model="text-embedding-ada-002", + redis_flush_size=None, # s3 Bucket, boto3 configuration s3_bucket_name: Optional[str] = None, s3_region_name: Optional[str] = None, s3_api_version: Optional[str] = None, - s3_path: Optional[str] = None, # if you wish to save to a spefic path + s3_path: Optional[str] = None, # if you wish to save to a specific path s3_use_ssl: Optional[bool] = True, s3_verify: Optional[Union[bool, str]] = None, s3_endpoint_url: Optional[str] = None, @@ -299,7 +333,11 @@ def __init__( s3_aws_secret_access_key: Optional[str] = None, s3_aws_session_token: Optional[str] = None, s3_config: Optional[Any] = None, - **kwargs, + + # disk cache params + disk_cache_dir=None, + + **kwargs ): ``` diff --git a/docs/my-website/docs/caching/local_caching.md b/docs/my-website/docs/caching/local_caching.md index d0e26e4bf9..81c4edcb82 100644 --- a/docs/my-website/docs/caching/local_caching.md +++ b/docs/my-website/docs/caching/local_caching.md @@ -40,7 +40,7 @@ cache = Cache() cache.add_cache(cache_key="test-key", result="1234") -cache.get_cache(cache_key="test-key) +cache.get_cache(cache_key="test-key") ``` ## Caching with Streaming diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3c968ea57f..82b5cfa0bc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -186,7 +186,7 @@ const sidebars = { `observability/telemetry`, ], }, - "caching/redis_cache", + "caching/all_caches", { type: "category", label: "Tutorials", From c0c244006f53b7b21f2b95d3470e8a2c3400100a Mon Sep 17 00:00:00 2001 From: Antonio Loison Date: Fri, 10 May 2024 12:34:05 +0200 Subject: [PATCH 9/9] deps: remove diskcache from dependencies and add install in docs --- docs/my-website/docs/caching/all_caches.md | 10 ++++++++++ poetry.lock | 14 +------------- pyproject.toml | 5 ----- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/docs/my-website/docs/caching/all_caches.md b/docs/my-website/docs/caching/all_caches.md index 8f4196856e..eb309f9b8b 100644 --- a/docs/my-website/docs/caching/all_caches.md +++ b/docs/my-website/docs/caching/all_caches.md @@ -178,6 +178,14 @@ response2 = completion( ### Quick Start +Install diskcache: + +```shell +pip install diskcache +``` + +Then you can use the disk cache as follows. + ```python import litellm from litellm import completion @@ -200,6 +208,8 @@ response2 = completion( ``` +If you run the code two times, response1 will use the cache from the first run that was stored in a cache file. + diff --git a/poetry.lock b/poetry.lock index 2911806cb3..f3f7d40d5b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -605,17 +605,6 @@ ssh = ["bcrypt (>=3.1.5)"] test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] -[[package]] -name = "diskcache" -version = "5.6.3" -description = "Disk Cache -- Disk and file backed persistent cache." -optional = true -python-versions = ">=3" -files = [ - {file = "diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19"}, - {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"}, -] - [[package]] name = "distro" version = "1.9.0" @@ -2679,11 +2668,10 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.link testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] [extras] -disk-caching = ["diskcache"] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-kms", "prisma", "resend"] proxy = ["PyJWT", "apscheduler", "backoff", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "orjson", "python-multipart", "pyyaml", "rq", "uvicorn"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "aa6283846b24f5703626c2de10459254166551f469c34e75b229adafb6985eba" +content-hash = "ff38be297294f084a739ef869d41d3d80f09c80e1d05d2963073d49790f33f37" diff --git a/pyproject.toml b/pyproject.toml index ca01141d2c..ecd5e97bc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ azure-identity = {version = "^1.15.0", optional = true} azure-keyvault-secrets = {version = "^4.8.0", optional = true} google-cloud-kms = {version = "^2.21.3", optional = true} resend = {version = "^0.8.0", optional = true} -diskcache = {version = "^5.6.3", optional = true} [tool.poetry.extras] proxy = [ @@ -66,10 +65,6 @@ extra_proxy = [ "resend" ] -disk_caching = [ - "diskcache" -] - [tool.poetry.scripts] litellm = 'litellm:run_server'