From 17c6a4e5328fb1a94751784c0b3d3683224c44a4 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Tue, 13 Aug 2024 16:07:41 -0700 Subject: [PATCH] Improving the proxy docs for configuring with vllm --- docs/my-website/docs/proxy/configs.md | 3 ++- docs/my-website/docs/proxy/quick_start.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 1620d11cad..1060db5e72 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -55,7 +55,8 @@ model_list: - model_name: vllm-models litellm_params: model: openai/facebook/opt-125m # the `openai/` prefix tells litellm it's openai compatible - api_base: http://0.0.0.0:4000 + api_base: http://0.0.0.0:4000/v1 + api_key: none rpm: 1440 model_info: version: 2 diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md index 2049f6bb26..8f8de2a9fa 100644 --- a/docs/my-website/docs/proxy/quick_start.md +++ b/docs/my-website/docs/proxy/quick_start.md @@ -243,7 +243,8 @@ model_list: - model_name: vllm-model litellm_params: model: openai/ - api_base: # e.g. http://0.0.0.0:3000 + api_base: # e.g. http://0.0.0.0:3000/v1 + api_key: ``` ### Run proxy with config