From 130815bb933bd9ce11c1e9ca8224fc9acc256eed Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 18:42:42 -0700 Subject: [PATCH 01/15] add litellm.create_fine_tuning_job --- litellm/fine_tuning/main.py | 182 ++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 litellm/fine_tuning/main.py diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py new file mode 100644 index 0000000000..6fa2bd9fba --- /dev/null +++ b/litellm/fine_tuning/main.py @@ -0,0 +1,182 @@ +""" +Main File for Fine Tuning API implementation + +https://platform.openai.com/docs/api-reference/fine-tuning + +- fine_tuning.jobs.create() +- fine_tuning.jobs.list() +- client.fine_tuning.jobs.list_events() +""" + +import asyncio +import contextvars +import os +from functools import partial +from typing import Any, Coroutine, Dict, Literal, Optional, Union + +import httpx + +import litellm +from litellm.llms.openai_fine_tuning.openai import ( + FineTuningJob, + FineTuningJobCreate, + OpenAIFineTuningAPI, +) +from litellm.types.llms.openai import Hyperparameters +from litellm.types.router import * +from litellm.utils import supports_httpx_timeout + +####### ENVIRONMENT VARIABLES ################### +openai_fine_tuning_instance = OpenAIFineTuningAPI() +################################################# + + +async def acreate_fine_tuning_job( + model: str, + training_file: str, + hyperparameters: Optional[Hyperparameters] = None, + suffix: Optional[str] = None, + validation_file: Optional[str] = None, + integrations: Optional[List[str]] = None, + seed: Optional[int] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> FineTuningJob: + """ + Async: Creates and executes a batch from an uploaded file of request + + LiteLLM Equivalent of POST: https://api.openai.com/v1/batches + """ + try: + loop = asyncio.get_event_loop() + kwargs["acreate_fine_tuning_job"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + create_fine_tuning_job, + model, + training_file, + hyperparameters, + suffix, + validation_file, + integrations, + seed, + custom_llm_provider, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + return response + except Exception as e: + raise e + + +def create_fine_tuning_job( + model: str, + training_file: str, + hyperparameters: Optional[Hyperparameters] = None, + suffix: Optional[str] = None, + validation_file: Optional[str] = None, + integrations: Optional[List[str]] = None, + seed: Optional[int] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: + """ + Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + + Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete + + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("acreate_fine_tuning_job", False) is True + + create_fine_tuning_job_data = FineTuningJobCreate( + model=model, + training_file=training_file, + hyperparameters=hyperparameters, + suffix=suffix, + validation_file=validation_file, + integrations=integrations, + seed=seed, + ) + + response = openai_fine_tuning_instance.create_fine_tuning_job( + api_base=api_base, + api_key=api_key, + organization=organization, + create_fine_tuning_job_data=create_fine_tuning_job_data, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e From 59fc3ba649acc6178ea4a078fc250bd47ce6d086 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 18:57:29 -0700 Subject: [PATCH 02/15] add create_fine_tuning --- litellm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 97a0a05ea9..72aeb74d99 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -906,6 +906,7 @@ from .proxy.proxy_cli import run_server from .router import Router from .assistants.main import * from .batches.main import * +from .fine_tuning.main import * from .files.main import * from .scheduler import * from .cost_calculator import response_cost_calculator, cost_per_token From 1202e1c645c1c6d6db33ed618593bd948cfc0cf1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 18:59:29 -0700 Subject: [PATCH 03/15] add acreate_fine_tuning_job --- litellm/fine_tuning/main.py | 4 +- litellm/llms/openai_fine_tuning/openai.py | 96 +++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/openai_fine_tuning/openai.py diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 6fa2bd9fba..de899fe4bd 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -34,7 +34,7 @@ openai_fine_tuning_instance = OpenAIFineTuningAPI() async def acreate_fine_tuning_job( model: str, training_file: str, - hyperparameters: Optional[Hyperparameters] = None, + hyperparameters: Optional[Hyperparameters] = {}, suffix: Optional[str] = None, validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, @@ -85,7 +85,7 @@ async def acreate_fine_tuning_job( def create_fine_tuning_job( model: str, training_file: str, - hyperparameters: Optional[Hyperparameters] = None, + hyperparameters: Optional[Hyperparameters] = {}, suffix: Optional[str] = None, validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py new file mode 100644 index 0000000000..c964108328 --- /dev/null +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -0,0 +1,96 @@ +from typing import Any, Coroutine, Optional, Union + +import httpx +from openai import AsyncOpenAI, OpenAI +from openai.types.fine_tuning import FineTuningJob + +from litellm._logging import verbose_logger +from litellm.llms.base import BaseLLM +from litellm.types.llms.openai import FineTuningJobCreate + + +class OpenAIFineTuningAPI(BaseLLM): + """ + OpenAI methods to support for batches + """ + + def __init__(self) -> None: + super().__init__() + + def get_openai_client( + self, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + _is_async: bool = False, + ) -> Optional[Union[OpenAI, AsyncOpenAI]]: + received_args = locals() + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = None + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client" or k == "_is_async": + pass + elif k == "api_base" and v is not None: + data["base_url"] = v + elif v is not None: + data[k] = v + if _is_async is True: + openai_client = AsyncOpenAI(**data) + else: + openai_client = OpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + + async def acreate_fine_tuning_job( + self, + create_fine_tuning_job_data: FineTuningJobCreate, + openai_client: AsyncOpenAI, + ) -> FineTuningJob: + response = await openai_client.batches.create(**create_fine_tuning_job_data) + return response + + def create_fine_tuning_job( + self, + _is_async: bool, + create_fine_tuning_job_data: FineTuningJobCreate, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ) -> Union[Coroutine[Any, Any, FineTuningJob]]: + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.acreate_fine_tuning_job( # type: ignore + create_fine_tuning_job_data=create_fine_tuning_job_data, + openai_client=openai_client, + ) + verbose_logger.debug( + "creating fine tuning job, args= %s", create_fine_tuning_job_data + ) + response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) + return response From 5123bf4e75c96d7e6f37ea51958cf68d7864df71 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 18:59:44 -0700 Subject: [PATCH 04/15] add types for FineTuningJobCreate OpenAI --- litellm/types/llms/openai.py | 78 +++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 35e442119d..fcff8b4bab 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -30,7 +30,7 @@ from openai.types.beta.thread_create_params import ( from openai.types.beta.threads.message import Message as OpenAIMessage from openai.types.beta.threads.message_content import MessageContent from openai.types.beta.threads.run import Run -from pydantic import BaseModel +from pydantic import BaseModel, Field from typing_extensions import Dict, Required, override FileContent = Union[IO[bytes], bytes, PathLike] @@ -455,3 +455,79 @@ class ChatCompletionUsageBlock(TypedDict): prompt_tokens: int completion_tokens: int total_tokens: int + + +class Hyperparameters(TypedDict): + batch_size: Optional[Union[str, int]] = Field( + default="auto", description="Number of examples in each batch." + ) + learning_rate_multiplier: Optional[Union[str, float]] = Field( + default="auto", description="Scaling factor for the learning rate." + ) + n_epochs: Optional[Union[str, int]] = Field( + default="auto", description="The number of epochs to train the model for." + ) + + +class FineTuningJobCreate(TypedDict): + """ + FineTuningJobCreate - Create a fine-tuning job + + Example Request + ``` + { + "model": "gpt-3.5-turbo", + "training_file": "file-abc123", + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": 0.1, + "n_epochs": 3 + }, + "suffix": "custom-model-name", + "validation_file": "file-xyz789", + "integrations": ["slack"], + "seed": 42 + } + ``` + """ + + model: str = Field(..., description="The name of the model to fine-tune.") + training_file: str = Field( + ..., description="The ID of an uploaded file that contains training data." + ) + hyperparameters: Optional[Hyperparameters] = Field( + default={}, description="The hyperparameters used for the fine-tuning job." + ) + suffix: Optional[str] = Field( + default=None, + description="A string of up to 18 characters that will be added to your fine-tuned model name.", + ) + validation_file: Optional[str] = Field( + default=None, + description="The ID of an uploaded file that contains validation data.", + ) + integrations: Optional[List[str]] = Field( + default=None, + description="A list of integrations to enable for your fine-tuning job.", + ) + seed: Optional[int] = Field( + default=None, description="The seed controls the reproducibility of the job." + ) + + class Config: + allow_population_by_field_name = True + schema_extra = { + "example": { + "model": "gpt-3.5-turbo", + "training_file": "file-abc123", + "hyperparameters": { + "batch_size": "auto", + "learning_rate_multiplier": 0.1, + "n_epochs": 3, + }, + "suffix": "custom-model-name", + "validation_file": "file-xyz789", + "integrations": ["slack"], + "seed": 42, + } + } From 3e3f9e3f0cbe746a13fef002adab1cb380c78f12 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 18:59:55 -0700 Subject: [PATCH 05/15] add test_create_fine_tune_job --- litellm/tests/test_fine_tuning_api.py | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 litellm/tests/test_fine_tuning_api.py diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py new file mode 100644 index 0000000000..eec81d0073 --- /dev/null +++ b/litellm/tests/test_fine_tuning_api.py @@ -0,0 +1,47 @@ +import os +import sys +import traceback + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +from openai import APITimeoutError as Timeout + +import litellm + +litellm.num_retries = 0 +import logging + +from litellm import create_fine_tuning_job +from litellm._logging import verbose_logger + + +def test_create_fine_tune_job(): + verbose_logger.setLevel(logging.DEBUG) + file_name = "openai_batch_completions.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + + file_obj = litellm.create_file( + file=open(file_path, "rb"), + purpose="fine-tune", + custom_llm_provider="openai", + ) + print("Response from creating file=", file_obj) + + response = litellm.create_fine_tuning_job( + model="gpt-3.5-turbo", + training_file=file_obj.id, + ) + + print("response from litellm.create_fine_tuning_job=", response) + + assert response.id is not None + assert response.model == "gpt-3.5-turbo" + + # delete file + + # cancel ft job + pass From 3802eaa6b558e28c788c435811a1588e64115fce Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:22:23 -0700 Subject: [PATCH 06/15] feat - add cancel_fine_tuning_job --- litellm/fine_tuning/main.py | 84 +++++++++++++++++++++++ litellm/llms/openai_fine_tuning/openai.py | 50 ++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index de899fe4bd..8784fdd9b4 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -180,3 +180,87 @@ def create_fine_tuning_job( return response except Exception as e: raise e + + +def cancel_fine_tuning_job( + fine_tuning_job_id: str, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: + """ + Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + + Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete + + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("acreate_fine_tuning_job", False) is True + + response = openai_fine_tuning_instance.cancel_fine_tuning_job( + api_base=api_base, + api_key=api_key, + organization=organization, + fine_tuning_job_id=fine_tuning_job_id, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py index c964108328..91924edab1 100644 --- a/litellm/llms/openai_fine_tuning/openai.py +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -94,3 +94,53 @@ class OpenAIFineTuningAPI(BaseLLM): ) response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) return response + + async def acancel_fine_tuning_job( + self, + fine_tuning_job_id: str, + openai_client: AsyncOpenAI, + ) -> FineTuningJob: + response = await openai_client.fine_tuning.jobs.cancel( + fine_tuning_job_id=fine_tuning_job_id + ) + return response + + def cancel_fine_tuning_job( + self, + _is_async: bool, + fine_tuning_job_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + ): + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.acancel_fine_tuning_job( # type: ignore + fine_tuning_job_id=fine_tuning_job_id, + openai_client=openai_client, + ) + verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id) + response = openai_client.fine_tuning.jobs.cancel( + fine_tuning_job_id=fine_tuning_job_id + ) + return response From 16d595c4ff0287400b220bab479faccbf3ffbf34 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:22:41 -0700 Subject: [PATCH 07/15] test cancel cancel_fine_tuning_job --- litellm/tests/test_fine_tuning_api.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index eec81d0073..d4935cb0d5 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -31,17 +31,29 @@ def test_create_fine_tune_job(): ) print("Response from creating file=", file_obj) - response = litellm.create_fine_tuning_job( - model="gpt-3.5-turbo", + create_fine_tuning_response = litellm.create_fine_tuning_job( + model="gpt-3.5-turbo-0125", training_file=file_obj.id, ) - print("response from litellm.create_fine_tuning_job=", response) + print("response from litellm.create_fine_tuning_job=", create_fine_tuning_response) - assert response.id is not None - assert response.model == "gpt-3.5-turbo" + assert create_fine_tuning_response.id is not None + assert create_fine_tuning_response.model == "gpt-3.5-turbo-0125" # delete file + litellm.file_delete( + file_id=file_obj.id, + ) + # cancel ft job + response = litellm.cancel_fine_tuning_job( + fine_tuning_job_id=create_fine_tuning_response.id, + ) + + print("response from litellm.cancel_fine_tuning_job=", response) + + assert response.status == "cancelled" + assert response.id == create_fine_tuning_response.id pass From 8e6df89f8aaac34a5da293bcee42c9d84affadeb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:23:35 -0700 Subject: [PATCH 08/15] fix doc string --- litellm/fine_tuning/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 8784fdd9b4..0866106840 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -190,7 +190,7 @@ def cancel_fine_tuning_job( **kwargs, ) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: """ - Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + Immediately cancel a fine-tune job. Response includes details of the enqueued job including job status and the name of the fine-tuned models once complete From 46772436f1403dfc39aec48b9edee2ba419592d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:25:36 -0700 Subject: [PATCH 09/15] async cancel ft job --- litellm/fine_tuning/main.py | 40 +++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 0866106840..e5f2a4555c 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -47,7 +47,6 @@ async def acreate_fine_tuning_job( """ Async: Creates and executes a batch from an uploaded file of request - LiteLLM Equivalent of POST: https://api.openai.com/v1/batches """ try: loop = asyncio.get_event_loop() @@ -182,6 +181,43 @@ def create_fine_tuning_job( raise e +async def acancel_fine_tuning_job( + fine_tuning_job_id: str, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> FineTuningJob: + """ + Async: Immediately cancel a fine-tune job. + """ + try: + loop = asyncio.get_event_loop() + kwargs["acancel_fine_tuning_job"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + cancel_fine_tuning_job, + fine_tuning_job_id, + custom_llm_provider, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + return response + except Exception as e: + raise e + + def cancel_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai"] = "openai", @@ -237,7 +273,7 @@ def cancel_fine_tuning_job( elif timeout is None: timeout = 600.0 - _is_async = kwargs.pop("acreate_fine_tuning_job", False) is True + _is_async = kwargs.pop("acancel_fine_tuning_job", False) is True response = openai_fine_tuning_instance.cancel_fine_tuning_job( api_base=api_base, From 4849df03ff551f38653e0704a9bc5ac9dec8e879 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:46:55 -0700 Subject: [PATCH 10/15] add list fine tune endpoints --- litellm/fine_tuning/main.py | 127 ++++++++++++++++++++++ litellm/llms/openai_fine_tuning/openai.py | 51 +++++++++ 2 files changed, 178 insertions(+) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index e5f2a4555c..8bb9bf1a53 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -300,3 +300,130 @@ def cancel_fine_tuning_job( return response except Exception as e: raise e + + +async def alist_fine_tuning_jobs( + after: Optional[str] = None, + limit: Optional[int] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +) -> FineTuningJob: + """ + Async: List your organization's fine-tuning jobs + """ + try: + loop = asyncio.get_event_loop() + kwargs["alist_fine_tuning_jobs"] = True + + # Use a partial function to pass your keyword arguments + func = partial( + cancel_fine_tuning_job, + after, + limit, + custom_llm_provider, + extra_headers, + extra_body, + **kwargs, + ) + + # Add the context to the function + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response # type: ignore + return response + except Exception as e: + raise e + + +def list_fine_tuning_jobs( + after: Optional[str] = None, + limit: Optional[int] = None, + custom_llm_provider: Literal["openai"] = "openai", + extra_headers: Optional[Dict[str, str]] = None, + extra_body: Optional[Dict[str, str]] = None, + **kwargs, +): + """ + List your organization's fine-tuning jobs + + Params: + + - after: Optional[str] = None, Identifier for the last job from the previous pagination request. + - limit: Optional[int] = None, Number of fine-tuning jobs to retrieve. Defaults to 20 + """ + try: + optional_params = GenericLiteLLMParams(**kwargs) + if custom_llm_provider == "openai": + + # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base = ( + optional_params.api_base + or litellm.api_base + or os.getenv("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + optional_params.organization + or litellm.organization + or os.getenv("OPENAI_ORGANIZATION", None) + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + optional_params.api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or os.getenv("OPENAI_API_KEY") + ) + ### TIMEOUT LOGIC ### + timeout = ( + optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + ) + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("alist_fine_tuning_jobs", False) is True + + response = openai_fine_tuning_instance.list_fine_tuning_jobs( + api_base=api_base, + api_key=api_key, + organization=organization, + after=after, + limit=limit, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) + else: + raise litellm.exceptions.BadRequestError( + message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( + custom_llm_provider + ), + model="n/a", + llm_provider=custom_llm_provider, + response=httpx.Response( + status_code=400, + content="Unsupported provider", + request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"), # type: ignore + ), + ) + return response + except Exception as e: + raise e diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py index 91924edab1..ee79483b79 100644 --- a/litellm/llms/openai_fine_tuning/openai.py +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -2,6 +2,7 @@ from typing import Any, Coroutine, Optional, Union import httpx from openai import AsyncOpenAI, OpenAI +from openai.pagination import AsyncCursorPage from openai.types.fine_tuning import FineTuningJob from litellm._logging import verbose_logger @@ -144,3 +145,53 @@ class OpenAIFineTuningAPI(BaseLLM): fine_tuning_job_id=fine_tuning_job_id ) return response + + async def alist_fine_tuning_jobs( + self, + openai_client: AsyncOpenAI, + after: Optional[str] = None, + limit: Optional[int] = None, + ): + response = await openai_client.fine_tuning.jobs.list(after=after, limit=limit) + return response + + def list_fine_tuning_jobs( + self, + _is_async: bool, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[OpenAI, AsyncOpenAI]] = None, + after: Optional[str] = None, + limit: Optional[int] = None, + ): + openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + if openai_client is None: + raise ValueError( + "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncOpenAI): + raise ValueError( + "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client." + ) + return self.alist_fine_tuning_jobs( # type: ignore + after=after, + limit=limit, + openai_client=openai_client, + ) + verbose_logger.debug("list fine tuning job, after= %s, limit= %s", after, limit) + response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) + return response + pass From 106626f2248b8219f34887489f773b6621568563 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:47:14 -0700 Subject: [PATCH 11/15] test - list_fine_tuning_jobs --- litellm/tests/test_fine_tuning_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index d4935cb0d5..4a3922697f 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -41,6 +41,12 @@ def test_create_fine_tune_job(): assert create_fine_tuning_response.id is not None assert create_fine_tuning_response.model == "gpt-3.5-turbo-0125" + # list fine tuning jobs + print("listing ft jobs") + ft_jobs = litellm.list_fine_tuning_jobs(limit=2) + print("response from litellm.list_fine_tuning_jobs=", ft_jobs) + assert len(ft_jobs) > 0 + # delete file litellm.file_delete( From c9bea3a879913700e99dbe897a00ba2275705bfc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 19:52:14 -0700 Subject: [PATCH 12/15] test - async ft jobs --- litellm/fine_tuning/main.py | 2 +- litellm/llms/openai_fine_tuning/openai.py | 4 +- litellm/tests/test_fine_tuning_api.py | 51 ++++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 8bb9bf1a53..b41ced1b90 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -319,7 +319,7 @@ async def alist_fine_tuning_jobs( # Use a partial function to pass your keyword arguments func = partial( - cancel_fine_tuning_job, + list_fine_tuning_jobs, after, limit, custom_llm_provider, diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py index ee79483b79..b955b9ce8d 100644 --- a/litellm/llms/openai_fine_tuning/openai.py +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -53,7 +53,9 @@ class OpenAIFineTuningAPI(BaseLLM): create_fine_tuning_job_data: FineTuningJobCreate, openai_client: AsyncOpenAI, ) -> FineTuningJob: - response = await openai_client.batches.create(**create_fine_tuning_job_data) + response = await openai_client.fine_tuning.jobs.create( + **create_fine_tuning_job_data + ) return response def create_fine_tuning_job( diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index 4a3922697f..b7e3c957cc 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -45,7 +45,8 @@ def test_create_fine_tune_job(): print("listing ft jobs") ft_jobs = litellm.list_fine_tuning_jobs(limit=2) print("response from litellm.list_fine_tuning_jobs=", ft_jobs) - assert len(ft_jobs) > 0 + + assert len(list(ft_jobs)) > 0 # delete file @@ -63,3 +64,51 @@ def test_create_fine_tune_job(): assert response.status == "cancelled" assert response.id == create_fine_tuning_response.id pass + + +@pytest.mark.asyncio +async def test_create_fine_tune_jobs_async(): + verbose_logger.setLevel(logging.DEBUG) + file_name = "openai_batch_completions.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="fine-tune", + custom_llm_provider="openai", + ) + print("Response from creating file=", file_obj) + + create_fine_tuning_response = await litellm.acreate_fine_tuning_job( + model="gpt-3.5-turbo-0125", + training_file=file_obj.id, + ) + + print("response from litellm.create_fine_tuning_job=", create_fine_tuning_response) + + assert create_fine_tuning_response.id is not None + assert create_fine_tuning_response.model == "gpt-3.5-turbo-0125" + + # list fine tuning jobs + print("listing ft jobs") + ft_jobs = await litellm.alist_fine_tuning_jobs(limit=2) + print("response from litellm.list_fine_tuning_jobs=", ft_jobs) + assert len(list(ft_jobs)) > 0 + + # delete file + + await litellm.afile_delete( + file_id=file_obj.id, + ) + + # cancel ft job + response = await litellm.acancel_fine_tuning_job( + fine_tuning_job_id=create_fine_tuning_response.id, + ) + + print("response from litellm.cancel_fine_tuning_job=", response) + + assert response.status == "cancelled" + assert response.id == create_fine_tuning_response.id + pass From 6abc49c6117d5a169bdba8490520c44d8dcbc2b8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 20:01:12 -0700 Subject: [PATCH 13/15] fix linting --- litellm/types/llms/openai.py | 39 ++++++++++++++---------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index fcff8b4bab..d6dbb8f5f1 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -458,15 +458,11 @@ class ChatCompletionUsageBlock(TypedDict): class Hyperparameters(TypedDict): - batch_size: Optional[Union[str, int]] = Field( - default="auto", description="Number of examples in each batch." - ) - learning_rate_multiplier: Optional[Union[str, float]] = Field( - default="auto", description="Scaling factor for the learning rate." - ) - n_epochs: Optional[Union[str, int]] = Field( - default="auto", description="The number of epochs to train the model for." - ) + batch_size: Optional[Union[str, int]] # "Number of examples in each batch." + learning_rate_multiplier: Optional[ + Union[str, float] + ] # Scaling factor for the learning rate + n_epochs: Optional[Union[str, int]] # "The number of epochs to train the model for" class FineTuningJobCreate(TypedDict): @@ -498,21 +494,16 @@ class FineTuningJobCreate(TypedDict): hyperparameters: Optional[Hyperparameters] = Field( default={}, description="The hyperparameters used for the fine-tuning job." ) - suffix: Optional[str] = Field( - default=None, - description="A string of up to 18 characters that will be added to your fine-tuned model name.", - ) - validation_file: Optional[str] = Field( - default=None, - description="The ID of an uploaded file that contains validation data.", - ) - integrations: Optional[List[str]] = Field( - default=None, - description="A list of integrations to enable for your fine-tuning job.", - ) - seed: Optional[int] = Field( - default=None, description="The seed controls the reproducibility of the job." - ) + suffix: Optional[ + str + ] # "A string of up to 18 characters that will be added to your fine-tuned model name." + validation_file: Optional[ + str + ] # "The ID of an uploaded file that contains validation data." + integrations: Optional[ + List[str] + ] # "A list of integrations to enable for your fine-tuning job." + seed: Optional[int] # "The seed controls the reproducibility of the job." class Config: allow_population_by_field_name = True From f18827cbc01125d459fa657a84aed9ea23f8be01 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 20:10:03 -0700 Subject: [PATCH 14/15] fix type errors --- litellm/fine_tuning/main.py | 4 +-- litellm/llms/openai_fine_tuning/openai.py | 8 +++--- litellm/types/llms/openai.py | 30 ++++------------------- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index b41ced1b90..eb5c7d4a43 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -34,7 +34,7 @@ openai_fine_tuning_instance = OpenAIFineTuningAPI() async def acreate_fine_tuning_job( model: str, training_file: str, - hyperparameters: Optional[Hyperparameters] = {}, + hyperparameters: Optional[Hyperparameters] = {}, # type: ignore suffix: Optional[str] = None, validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, @@ -84,7 +84,7 @@ async def acreate_fine_tuning_job( def create_fine_tuning_job( model: str, training_file: str, - hyperparameters: Optional[Hyperparameters] = {}, + hyperparameters: Optional[Hyperparameters] = {}, # type: ignore suffix: Optional[str] = None, validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py index b955b9ce8d..d81ed37601 100644 --- a/litellm/llms/openai_fine_tuning/openai.py +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -54,7 +54,7 @@ class OpenAIFineTuningAPI(BaseLLM): openai_client: AsyncOpenAI, ) -> FineTuningJob: response = await openai_client.fine_tuning.jobs.create( - **create_fine_tuning_job_data + **create_fine_tuning_job_data # type: ignore ) return response @@ -68,7 +68,7 @@ class OpenAIFineTuningAPI(BaseLLM): max_retries: Optional[int], organization: Optional[str], client: Optional[Union[OpenAI, AsyncOpenAI]] = None, - ) -> Union[Coroutine[Any, Any, FineTuningJob]]: + ) -> Union[FineTuningJob, Union[Coroutine[Any, Any, FineTuningJob]]]: openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client( api_key=api_key, api_base=api_base, @@ -154,7 +154,7 @@ class OpenAIFineTuningAPI(BaseLLM): after: Optional[str] = None, limit: Optional[int] = None, ): - response = await openai_client.fine_tuning.jobs.list(after=after, limit=limit) + response = await openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore return response def list_fine_tuning_jobs( @@ -194,6 +194,6 @@ class OpenAIFineTuningAPI(BaseLLM): openai_client=openai_client, ) verbose_logger.debug("list fine tuning job, after= %s, limit= %s", after, limit) - response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) + response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore return response pass diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index d6dbb8f5f1..396e58e994 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -487,13 +487,11 @@ class FineTuningJobCreate(TypedDict): ``` """ - model: str = Field(..., description="The name of the model to fine-tune.") - training_file: str = Field( - ..., description="The ID of an uploaded file that contains training data." - ) - hyperparameters: Optional[Hyperparameters] = Field( - default={}, description="The hyperparameters used for the fine-tuning job." - ) + model: str # "The name of the model to fine-tune." + training_file: str # "The ID of an uploaded file that contains training data." + hyperparameters: Optional[ + Hyperparameters + ] # "The hyperparameters used for the fine-tuning job." suffix: Optional[ str ] # "A string of up to 18 characters that will be added to your fine-tuned model name." @@ -504,21 +502,3 @@ class FineTuningJobCreate(TypedDict): List[str] ] # "A list of integrations to enable for your fine-tuning job." seed: Optional[int] # "The seed controls the reproducibility of the job." - - class Config: - allow_population_by_field_name = True - schema_extra = { - "example": { - "model": "gpt-3.5-turbo", - "training_file": "file-abc123", - "hyperparameters": { - "batch_size": "auto", - "learning_rate_multiplier": 0.1, - "n_epochs": 3, - }, - "suffix": "custom-model-name", - "validation_file": "file-xyz789", - "integrations": ["slack"], - "seed": 42, - } - } From dff8163f2cc3e25a037a8283a383c687382f53d9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 29 Jul 2024 20:10:33 -0700 Subject: [PATCH 15/15] fix type errors --- litellm/llms/openai_fine_tuning/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/openai_fine_tuning/openai.py index d81ed37601..2f6d89ea0b 100644 --- a/litellm/llms/openai_fine_tuning/openai.py +++ b/litellm/llms/openai_fine_tuning/openai.py @@ -95,7 +95,7 @@ class OpenAIFineTuningAPI(BaseLLM): verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) - response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) + response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) # type: ignore return response async def acancel_fine_tuning_job(