{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Ys9n20Es2IzT"
      },
      "source": [
        "# Evaluate Multiple LLM Providers with LiteLLM\n",
        "\n",
        "\n",
        "\n",
        "*   Quality Testing\n",
        "*   Load Testing\n",
        "*   Duration Testing\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZXOXl23PIIP6"
      },
      "outputs": [],
      "source": [
        "!pip install litellm python-dotenv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "LINuBzXDItq2"
      },
      "outputs": [],
      "source": [
        "import litellm\n",
        "from litellm import load_test_model, testing_batch_completion\n",
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EkxMhsWdJdu4"
      },
      "outputs": [],
      "source": [
        "import os \n",
        "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n",
        "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n",
        "os.environ[\"REPLICATE_API_KEY\"] = \"...\""
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mv5XdnqeW5I_"
      },
      "source": [
        "# Quality Test endpoint\n",
        "\n",
        "## Test the same prompt across multiple LLM providers\n",
        "\n",
        "In this example, let's ask some questions about Paul Graham"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "XpzrR5m4W_Us"
      },
      "outputs": [],
      "source": [
        "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", {\"model\": \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"custom_llm_provider\": \"replicate\"}]\n",
        "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
        "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n",
        "messages =  [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n",
        "result = testing_batch_completion(models=models, messages=messages)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "9nzeLySnvIIW"
      },
      "source": [
        "## Visualize the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 403
        },
        "id": "X-2n7hdAuVAY",
        "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-8c39923a-ebb1-42ef-b7a0-5edd2535cb37\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>Model Name</th>\n",
              "      <th>claude-instant-1</th>\n",
              "      <th>gpt-3.5-turbo-0613</th>\n",
              "      <th>gpt-3.5-turbo-16k-0613</th>\n",
              "      <th>gpt-4-0613</th>\n",
              "      <th>replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Prompt</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>\\nIs paul graham a writer?</th>\n",
              "      <td>Yes, Paul Graham is considered a writer in ad...</td>\n",
              "      <td>Yes, Paul Graham is a writer. He has written s...</td>\n",
              "      <td>Yes, Paul Graham is a writer. He has authored ...</td>\n",
              "      <td>Yes, Paul Graham is a writer. He is an essayis...</td>\n",
              "      <td>Yes, Paul Graham is an author. According to t...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>\\nWhat has Paul Graham done?</th>\n",
              "      <td>Paul Graham has made significant contribution...</td>\n",
              "      <td>Paul Graham has achieved several notable accom...</td>\n",
              "      <td>Paul Graham has made significant contributions...</td>\n",
              "      <td>Paul Graham is known for his work on the progr...</td>\n",
              "      <td>Paul Graham has had a diverse career in compu...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>\\nWhat is Paul Graham known for?</th>\n",
              "      <td>Paul Graham is known for several things:\\n\\n-...</td>\n",
              "      <td>Paul Graham is known for his work on the progr...</td>\n",
              "      <td>Paul Graham is known for his work on the progr...</td>\n",
              "      <td>Paul Graham is known for his work on the progr...</td>\n",
              "      <td>Paul Graham is known for many things, includi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>\\nWhere does Paul Graham live?</th>\n",
              "      <td>Based on the information provided:\\n\\n- Paul ...</td>\n",
              "      <td>According to the given information, Paul Graha...</td>\n",
              "      <td>Paul Graham currently lives in England, where ...</td>\n",
              "      <td>The text does not provide a current place of r...</td>\n",
              "      <td>Based on the information provided, Paul Graha...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>\\nWho is Paul Graham?</th>\n",
              "      <td>Paul Graham is an influential computer scient...</td>\n",
              "      <td>Paul Graham is an English computer scientist, ...</td>\n",
              "      <td>Paul Graham is an English computer scientist, ...</td>\n",
              "      <td>Paul Graham is an English computer scientist, ...</td>\n",
              "      <td>Paul Graham is an English computer scientist,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8c39923a-ebb1-42ef-b7a0-5edd2535cb37')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-4d5c5cee-4f56-4ad2-b181-59c3ec519d1f\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4d5c5cee-4f56-4ad2-b181-59c3ec519d1f')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-4d5c5cee-4f56-4ad2-b181-59c3ec519d1f button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8c39923a-ebb1-42ef-b7a0-5edd2535cb37 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8c39923a-ebb1-42ef-b7a0-5edd2535cb37');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "Model Name                                                         claude-instant-1  \\\n",
              "Prompt                                                                                \n",
              "\\nIs paul graham a writer?         Yes, Paul Graham is considered a writer in ad...   \n",
              "\\nWhat has Paul Graham done?       Paul Graham has made significant contribution...   \n",
              "\\nWhat is Paul Graham known for?   Paul Graham is known for several things:\\n\\n-...   \n",
              "\\nWhere does Paul Graham live?     Based on the information provided:\\n\\n- Paul ...   \n",
              "\\nWho is Paul Graham?              Paul Graham is an influential computer scient...   \n",
              "\n",
              "Model Name                                                       gpt-3.5-turbo-0613  \\\n",
              "Prompt                                                                                \n",
              "\\nIs paul graham a writer?        Yes, Paul Graham is a writer. He has written s...   \n",
              "\\nWhat has Paul Graham done?      Paul Graham has achieved several notable accom...   \n",
              "\\nWhat is Paul Graham known for?  Paul Graham is known for his work on the progr...   \n",
              "\\nWhere does Paul Graham live?    According to the given information, Paul Graha...   \n",
              "\\nWho is Paul Graham?             Paul Graham is an English computer scientist, ...   \n",
              "\n",
              "Model Name                                                   gpt-3.5-turbo-16k-0613  \\\n",
              "Prompt                                                                                \n",
              "\\nIs paul graham a writer?        Yes, Paul Graham is a writer. He has authored ...   \n",
              "\\nWhat has Paul Graham done?      Paul Graham has made significant contributions...   \n",
              "\\nWhat is Paul Graham known for?  Paul Graham is known for his work on the progr...   \n",
              "\\nWhere does Paul Graham live?    Paul Graham currently lives in England, where ...   \n",
              "\\nWho is Paul Graham?             Paul Graham is an English computer scientist, ...   \n",
              "\n",
              "Model Name                                                               gpt-4-0613  \\\n",
              "Prompt                                                                                \n",
              "\\nIs paul graham a writer?        Yes, Paul Graham is a writer. He is an essayis...   \n",
              "\\nWhat has Paul Graham done?      Paul Graham is known for his work on the progr...   \n",
              "\\nWhat is Paul Graham known for?  Paul Graham is known for his work on the progr...   \n",
              "\\nWhere does Paul Graham live?    The text does not provide a current place of r...   \n",
              "\\nWho is Paul Graham?             Paul Graham is an English computer scientist, ...   \n",
              "\n",
              "Model Name                       replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781  \n",
              "Prompt                                                                                                                        \n",
              "\\nIs paul graham a writer?         Yes, Paul Graham is an author. According to t...                                           \n",
              "\\nWhat has Paul Graham done?       Paul Graham has had a diverse career in compu...                                           \n",
              "\\nWhat is Paul Graham known for?   Paul Graham is known for many things, includi...                                           \n",
              "\\nWhere does Paul Graham live?     Based on the information provided, Paul Graha...                                           \n",
              "\\nWho is Paul Graham?              Paul Graham is an English computer scientist,...                                           "
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Create an empty list to store the row data\n",
        "table_data = []\n",
        "\n",
        "# Iterate through the list and extract the required data\n",
        "for item in result:\n",
        "    prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n",
        "    model = item['response']['model']\n",
        "    response = item['response']['choices'][0]['message']['content']\n",
        "    table_data.append([prompt, model, response])\n",
        "\n",
        "# Create a DataFrame from the table data\n",
        "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n",
        "\n",
        "# Pivot the DataFrame to get the desired table format\n",
        "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n",
        "table"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zOxUM40PINDC"
      },
      "source": [
        "# Load Test endpoint\n",
        "\n",
        "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZkQf_wbcIRQ9"
      },
      "outputs": [],
      "source": [
        "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n",
        "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
        "prompt = \"Where does Paul Graham live?\"\n",
        "final_prompt = context + prompt\n",
        "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "8vSNBFC06aXY"
      },
      "source": [
        "## Visualize the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        },
        "id": "SZfiKjLV3-n8",
        "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 35660 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "## calculate avg response time\n",
        "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n",
        "model_dict = {model: {\"response_time\": []} for model in unique_models}\n",
        "for completion_result in result[\"results\"]:\n",
        "    model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n",
        "\n",
        "avg_response_time = {}\n",
        "for model, data in model_dict.items():\n",
        "    avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n",
        "\n",
        "models = list(avg_response_time.keys())\n",
        "response_times = list(avg_response_time.values())\n",
        "\n",
        "plt.bar(models, response_times)\n",
        "plt.xlabel('Model', fontsize=10)\n",
        "plt.ylabel('Average Response Time')\n",
        "plt.title('Average Response Times for each Model')\n",
        "\n",
        "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n",
        "plt.show()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "inSDIE3_IRds"
      },
      "source": [
        "# Duration Test endpoint\n",
        "\n",
        "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "ePIqDx2EIURH"
      },
      "outputs": [],
      "source": [
        "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n",
        "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n",
        "prompt = \"Where does Paul Graham live?\"\n",
        "final_prompt = context + prompt\n",
        "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 552
        },
        "id": "k6rJoELM6t1K",
        "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7"
      },
      "outputs": [
        {
          "data": {
            "image/png": "[encoded data, 38584 characters]",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "## calculate avg response time\n",
        "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n",
        "model_dict = {model: {\"response_time\": []} for model in unique_models}\n",
        "for iteration in result:\n",
        "  for completion_result in iteration[\"results\"]:\n",
        "    model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n",
        "\n",
        "avg_response_time = {}\n",
        "for model, data in model_dict.items():\n",
        "    avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n",
        "\n",
        "models = list(avg_response_time.keys())\n",
        "response_times = list(avg_response_time.values())\n",
        "\n",
        "plt.bar(models, response_times)\n",
        "plt.xlabel('Model', fontsize=10)\n",
        "plt.ylabel('Average Response Time')\n",
        "plt.title('Average Response Times for each Model')\n",
        "\n",
        "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
