litellm/cookbook/Parallel_function_calling.ipynb

This is a tutorial on using Parallel function calling with LiteLLM

In [ ]:

!pip install litellm

This tutorial walks through the steps doing parallel function calling using

• OpenAI
• Azure OpenAI

In [3]:

# set openai api key
import os
os.environ['OPENAI_API_KEY'] = "" # litellm reads OPENAI_API_KEY from .env and sends the request

OpenAI gpt-3.5-turbo-1106

Step 1: send the conversation and available functions to the model

In [18]:

import litellm
import json
# Example dummy function hard coded to return the same weather
# In production, this could be your backend API or an external API
def get_current_weather(location, unit="fahrenheit"):
    """Get the current weather in a given location"""
    if "tokyo" in location.lower():
        return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
    elif "san francisco" in location.lower():
        return json.dumps({"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"})
    elif "paris" in location.lower():
        return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
    else:
        return json.dumps({"location": location, "temperature": "unknown"})

messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}]
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_current_weather",
            "description": "Get the current weather in a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA",
                    },
                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
                },
                "required": ["location"],
            },
        },
    }
]

response = litellm.completion(
    model="gpt-3.5-turbo-1106",
    messages=messages,
    tools=tools,
    tool_choice="auto",  # auto is default, but we'll be explicit
)
print("\nLLM Response1:\n", response)
response_message = response.choices[0].message
tool_calls = response.choices[0].message.tool_calls
print("\nTool Choice:\n", tool_calls)

LLM Response1:
 ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{"location": "San Francisco", "unit": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{"location": "Tokyo", "unit": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{"location": "Paris", "unit": "celsius"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)

Tool Choice:
 [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{"location": "San Francisco", "unit": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{"location": "Tokyo", "unit": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{"location": "Paris", "unit": "celsius"}', name='get_current_weather'), type='function')]

Step 2 - Parse the Model Response and Execute Functions

In [21]:

# Check if the model wants to call a function
if tool_calls:
    # Execute the functions and prepare responses
    available_functions = {
        "get_current_weather": get_current_weather,
    }

    messages.append(response_message)  # Extend conversation with assistant's reply

    for tool_call in tool_calls:
      print(f"\nExecuting tool call\n{tool_call}")
      function_name = tool_call.function.name
      function_to_call = available_functions[function_name]
      function_args = json.loads(tool_call.function.arguments)
      # calling the get_current_weather() function
      function_response = function_to_call(
          location=function_args.get("location"),
          unit=function_args.get("unit"),
      )
      print(f"Result from tool call\n{function_response}\n")

      # Extend conversation with function response
      messages.append(
          {
              "tool_call_id": tool_call.id,
              "role": "tool",
              "name": function_name,
              "content": function_response,
          }
      )

Executing tool call
ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{"location": "San Francisco", "unit": "celsius"}', name='get_current_weather'), type='function')
Result from tool call
{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}


Executing tool call
ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{"location": "Tokyo", "unit": "celsius"}', name='get_current_weather'), type='function')
Result from tool call
{"location": "Tokyo", "temperature": "10", "unit": "celsius"}


Executing tool call
ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{"location": "Paris", "unit": "celsius"}', name='get_current_weather'), type='function')
Result from tool call
{"location": "Paris", "temperature": "22", "unit": "celsius"}

Step 3 - Second litellm.completion() call

In [26]:

second_response = litellm.completion(
    model="gpt-3.5-turbo-1106",
    messages=messages,
)
print("Second Response\n", second_response)
print("Second Response Message\n", second_response.choices[0].message.content)

Second Response
 ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content="The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)
Second Response Message
 The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.

Using Azure OpenAI

In [32]:

# set Azure env variables
import os
os.environ['AZURE_API_KEY'] = "" # litellm reads AZURE_API_KEY from .env and sends the request
os.environ['AZURE_API_BASE'] = "https://openai-gpt-4-test-v-1.openai.azure.com/"
os.environ['AZURE_API_VERSION'] = "2023-07-01-preview"

Step 1

In [33]:

import litellm
import json
# Example dummy function hard coded to return the same weather
# In production, this could be your backend API or an external API
def get_current_weather(location, unit="fahrenheit"):
    """Get the current weather in a given location"""
    if "tokyo" in location.lower():
        return json.dumps({"location": "Tokyo", "temperature": "10", "unit": "celsius"})
    elif "san francisco" in location.lower():
        return json.dumps({"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"})
    elif "paris" in location.lower():
        return json.dumps({"location": "Paris", "temperature": "22", "unit": "celsius"})
    else:
        return json.dumps({"location": location, "temperature": "unknown"})

messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}]
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_current_weather",
            "description": "Get the current weather in a given location",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA",
                    },
                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
                },
                "required": ["location"],
            },
        },
    }
]

response = litellm.completion(
    model="azure/chatgpt-functioncalling", # model = azure/<your-azure-deployment-name>
    messages=messages,
    tools=tools,
    tool_choice="auto",  # auto is default, but we'll be explicit
)
print("\nLLM Response1:\n", response)
response_message = response.choices[0].message
tool_calls = response.choices[0].message.tool_calls
print("\nTool Choice:\n", tool_calls)

LLM Response1:
 ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\n  "location": "San Francisco, CA"\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)

Tool Choice:
 [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\n  "location": "San Francisco, CA"\n}', name='get_current_weather'), type='function')]

Step 2

In [34]:

# Check if the model wants to call a function
if tool_calls:
    # Execute the functions and prepare responses
    available_functions = {
        "get_current_weather": get_current_weather,
    }

    messages.append(response_message)  # Extend conversation with assistant's reply

    for tool_call in tool_calls:
      print(f"\nExecuting tool call\n{tool_call}")
      function_name = tool_call.function.name
      function_to_call = available_functions[function_name]
      function_args = json.loads(tool_call.function.arguments)
      # calling the get_current_weather() function
      function_response = function_to_call(
          location=function_args.get("location"),
          unit=function_args.get("unit"),
      )
      print(f"Result from tool call\n{function_response}\n")

      # Extend conversation with function response
      messages.append(
          {
              "tool_call_id": tool_call.id,
              "role": "tool",
              "name": function_name,
              "content": function_response,
          }
      )

Executing tool call
ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\n  "location": "San Francisco, CA"\n}', name='get_current_weather'), type='function')
Result from tool call
{"location": "San Francisco", "temperature": "72", "unit": "fahrenheit"}

Step 3

In [36]:

second_response = litellm.completion(
    model="azure/chatgpt-functioncalling",
    messages=messages,
)
print("Second Response\n", second_response)
print("Second Response Message\n", second_response.choices[0].message.content)

Second Response
 ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)
Second Response Message
 The current weather in San Francisco is 72°F.