litellm/cookbook/VLLM_Model_Testing.ipynb

Set up Environment

In [1]:

!pip install --upgrade litellm

Collecting litellm
  Downloading litellm-0.1.555-py3-none-any.whl (100 kB)
[?25l     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0.0/100.4 kB ? eta -:--:--
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╸━━━ 92.2/100.4 kB 2.7 MB/s eta 0:00:01
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100.4/100.4 kB 2.3 MB/s eta 0:00:00
[?25hRequirement already satisfied: importlib-metadata<7.0.0,>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm) (6.8.0)
Collecting openai<0.28.0,>=0.27.8 (from litellm)
  Downloading openai-0.27.10-py3-none-any.whl (76 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 76.5/76.5 kB 8.9 MB/s eta 0:00:00
[?25hCollecting python-dotenv>=0.2.0 (from litellm)
  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)
Collecting tiktoken<0.5.0,>=0.4.0 (from litellm)
  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 32.3 MB/s eta 0:00:00
[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0.0,>=6.8.0->litellm) (3.16.2)
Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (2.31.0)
Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (4.66.1)
Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (3.8.5)
Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm) (2023.6.3)
Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.2.0)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2023.7.22)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (23.1.0)
Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (6.0.4)
Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (4.0.3)
Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.9.2)
Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.4.0)
Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.3.1)
Installing collected packages: python-dotenv, tiktoken, openai, litellm
Successfully installed litellm-0.1.555 openai-0.27.10 python-dotenv-1.0.0 tiktoken-0.4.0

In [2]:

!pip install vllm

Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21

Load the Logs

In [4]:

import pandas as pd

In [6]:

# path of the csv file
file_path = 'Model-prompts-example.csv'

# load the csv file as a pandas DataFrame
data = pd.read_csv(file_path)

data.head()

Out [6]:

	Success	Timestamp	Input	Output	RunId (Wandb Runid)	Model ID (or Name)
0	True	1694041195	This is the templated query input	This is the query output from the model	8hlumwuk	OpenAI/Turbo-3.5

In [7]:

input_texts = data['Input'].values

In [8]:

messages = [[{"role": "user", "content": input_text}] for input_text in input_texts]

Running Inference

In [ ]:

from litellm import batch_completion
model_name = "facebook/opt-125m"
provider = "vllm"
response_list = batch_completion(
            model=model_name,
            custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.
            messages=messages,
            temperature=0.2,
            max_tokens=80,
        )

In [10]:

response_list

Out [10]:

[<ModelResponse at 0x7e5b87616750> JSON: {
   "choices": [
     {
       "finish_reason": "stop",
       "index": 0,
       "message": {
         "content": ".\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is",
         "role": "assistant",
         "logprobs": null
       }
     }
   ],
   "created": 1694053363.6139505,
   "model": "facebook/opt-125m",
   "usage": {
     "prompt_tokens": 9,
     "completion_tokens": 80,
     "total_tokens": 89
   }
 }]

In [11]:

response_values = [response['choices'][0]['message']['content'] for response in response_list]

In [12]:

response_values

Out [12]:

['.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is used to query the data.\n\nThe query input is the query input that is']

In [13]:

data[f"{model_name}_output"] = response_values

In [14]:

data.to_csv('model_responses.csv', index=False)