From 8a2be96c674f4f89fd168ff37a76a87e03aef2b5 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Fri, 1 Sep 2023 20:21:54 -0700
Subject: [PATCH] ab test llms in production

---
 cookbook/litellm_ab_testllms.py | 58 +++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 cookbook/litellm_ab_testllms.py

diff --git a/cookbook/litellm_ab_testllms.py b/cookbook/litellm_ab_testllms.py
new file mode 100644
index 0000000000..ee77442c57
--- /dev/null
+++ b/cookbook/litellm_ab_testllms.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""LiteLLM_AB_TestLLMs.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1Y7zcpuJT5rqJYEELeg9UfP1Zf8gFVR_o
+
+# LiteLLM A/B Testing LLMs in production
+
+* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo`
+
+This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production
+
+# Example 1: A/B Test GPT-4 & GPT-3.5
+
+# Step 1
+👉 Get your `id` from here: https://admin.litellm.ai/
+"""
+
+from litellm import completion_with_split_tests
+import os
+
+## set ENV variables
+os.environ["OPENAI_API_KEY"] = "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"
+
+
+# define a dict of model id and % of requests for model
+# see models here: https://docs.litellm.ai/docs/completion/supported
+split_per_model = {
+	"gpt-4": 0.3,
+	"gpt-3.5-turbo": 0.7
+}
+
+messages = [{ "content": "Hello, how are you?","role": "user"}]
+
+completion_with_split_tests(messages=messages, use_client=True,
+   id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id
+
+"""## A/B Test GPT-4 and Claude-2"""
+
+from litellm import completion_with_split_tests
+import os
+
+## set ENV variables
+os.environ["ANTHROPIC_API_KEY"] = ""
+
+# define a dict of model id and % of requests for model
+split_per_model = {
+	"gpt-4": 0.3,
+	"claude-2": 0.7
+}
+
+messages = [{ "content": "Hello, how are you?","role": "user"}]
+
+
+completion_with_split_tests(messages=messages, use_client=True,
+   id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id
\ No newline at end of file