litellm/cookbook/VLLM_Model_Testing.ipynb

Set up Environment

In [ ]:

!pip install --upgrade litellm

In [2]:

!pip install vllm

Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21

Load the Logs

In [4]:

import pandas as pd

In [6]:

# path of the csv file
file_path = 'Model-prompts-example.csv'

# load the csv file as a pandas DataFrame
data = pd.read_csv(file_path)

data.head()

Out [6]:

	Success	Timestamp	Input	Output	RunId (Wandb Runid)	Model ID (or Name)
0	True	1694041195	This is the templated query input	This is the query output from the model	8hlumwuk	OpenAI/Turbo-3.5

In [7]:

input_texts = data['Input'].values

In [8]:

messages = [[{"role": "user", "content": input_text}] for input_text in input_texts]

Running Inference

In [ ]:

from litellm import batch_completion
model_name = "facebook/opt-125m"
provider = "vllm"
response_list = batch_completion(
            model=model_name,
            custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.
            messages=messages,
            temperature=0.2,
            max_tokens=80,
        )

In [10]:

response_list

Out [10]:

[<ModelResponse at 0x7e5b87616750> JSON: {
   "choices": [
     {
       "finish_reason": "stop",
       "index": 0,
       "message": {
         "content": ".\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is",
         "role": "assistant",
         "logprobs": null
       }
     }
   ],
   "created": 1694053363.6139505,
   "model": "facebook/opt-125m",
   "usage": {
     "prompt_tokens": 9,
     "completion_tokens": 80,
     "total_tokens": 89
   }
 }]

In [11]:

response_values = [response['choices'][0]['message']['content'] for response in response_list]

In [12]:

response_values

Out [12]:

['.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is']

In [13]:

data[f"{model_name}_output"] = response_values

In [14]:

data.to_csv('model_responses.csv', index=False)