mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-05 08:23:17 +00:00
docs(scheduler.md): add scheduler redis caching support to docs
This commit is contained in:
@@ -100,4 +100,76 @@ print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
</Tabs>
|
||||
|
||||
## Advanced - Redis Caching
|
||||
|
||||
Use redis caching to do request prioritization across multiple instances of LiteLLM.
|
||||
|
||||
### SDK
|
||||
```python
|
||||
from litellm import Router
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"mock_response": "Hello world this is Macintosh!", # fakes the LLM API call
|
||||
"rpm": 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
### REDIS PARAMS ###
|
||||
redis_host=os.environ["REDIS_HOST"],
|
||||
redis_password=os.environ["REDIS_PASSWORD"],
|
||||
redis_port=os.environ["REDIS_PORT"],
|
||||
)
|
||||
|
||||
try:
|
||||
_response = await router.schedule_acompletion( # 👈 ADDS TO QUEUE + POLLS + MAKES CALL
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hey!"}],
|
||||
priority=0, # 👈 LOWER IS BETTER
|
||||
)
|
||||
except Exception as e:
|
||||
print("didn't make request")
|
||||
```
|
||||
|
||||
### PROXY
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo-fake-model
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
mock_response: "hello world!"
|
||||
api_key: my-good-key
|
||||
|
||||
router_settings:
|
||||
redis_host; os.environ/REDIS_HOST
|
||||
redis_password: os.environ/REDIS_PASSWORD
|
||||
redis_port: os.environ/REDIS_PORT
|
||||
```
|
||||
|
||||
```bash
|
||||
$ litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000s
|
||||
```
|
||||
|
||||
```bash
|
||||
curl -X POST 'http://localhost:4000/queue/chat/completions' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-D '{
|
||||
"model": "gpt-3.5-turbo-fake-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "what is the meaning of the universe? 1234"
|
||||
}],
|
||||
"priority": 0 👈 SET VALUE HERE
|
||||
}'
|
||||
```
|
||||
Generated
+3
-3
@@ -5975,9 +5975,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/caniuse-lite": {
|
||||
"version": "1.0.30001519",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001519.tgz",
|
||||
"integrity": "sha512-0QHgqR+Jv4bxHMp8kZ1Kn8CH55OikjKJ6JmKkZYP1F3D7w+lnFXF70nG5eNfsZS89jadi5Ywy5UCSKLAglIRkg==",
|
||||
"version": "1.0.30001629",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001629.tgz",
|
||||
"integrity": "sha512-c3dl911slnQhmxUIT4HhYzT7wnBK/XYpGnYLOj4nJBaRiw52Ibe7YxlDaAeRECvA786zCuExhxIUJ2K7nHMrBw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "opencollective",
|
||||
|
||||
+961
-538
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user