Merge pull request #5184 from fozziethebeat/fix-vllm-docs

Improving the proxy docs for configuring with vllm
This commit is contained in:
Ishaan Jaff
2024-08-13 18:47:01 -07:00
committed by GitHub
2 changed files with 4 additions and 2 deletions
+2 -1
View File
@@ -55,7 +55,8 @@ model_list:
- model_name: vllm-models
litellm_params:
model: openai/facebook/opt-125m # the `openai/` prefix tells litellm it's openai compatible
api_base: http://0.0.0.0:4000
api_base: http://0.0.0.0:4000/v1
api_key: none
rpm: 1440
model_info:
version: 2
+2 -1
View File
@@ -243,7 +243,8 @@ model_list:
- model_name: vllm-model
litellm_params:
model: openai/<your-model-name>
api_base: <your-api-base> # e.g. http://0.0.0.0:3000
api_base: <your-vllm-api-base> # e.g. http://0.0.0.0:3000/v1
api_key: <your-vllm-api-key|none>
```
### Run proxy with config