Merge pull request #5224 from BerriAI/litellm_use_common_helper_llm_call

[Feat-Proxy] - user common helper to `route_request` for making llm call

litellm/proxy/proxy_config.yaml

@@ -19,6 +19,9 @@ model_list:
     litellm_params:
       model: mistral/mistral-small-latest
       api_key: "os.environ/MISTRAL_API_KEY"
+  - model_name: bedrock-anthropic
+    litellm_params:
+      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
   - model_name: gemini-1.5-pro-001
     litellm_params:
       model: vertex_ai_beta/gemini-1.5-pro-001
@@ -40,3 +43,6 @@ general_settings:
 litellm_settings:
   fallbacks: [{"gemini-1.5-pro-001": ["gpt-4o"]}]
   callbacks: ["gcs_bucket"]
+  success_callback: ["langfuse"]
+  langfuse_default_tags: ["cache_hit", "cache_key", "user_api_key_alias", "user_api_key_team_alias"]
+  cache: True

litellm/proxy/proxy_server.py

@@ -187,6 +187,7 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_confi
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     initialize_pass_through_endpoints,
 )
+from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.secret_managers.aws_secret_manager import (
     load_aws_kms,
     load_aws_secret_manager,
@@ -3006,68 +3007,13 @@ async def chat_completion(
 
         ### ROUTE THE REQUEST ###
         # Do not change this - it should be a constant time fetch - ALWAYS
-        router_model_names = llm_router.model_names if llm_router is not None else []
-        # skip router if user passed their key
-        if "api_key" in data:
-            tasks.append(litellm.acompletion(**data))
-        elif "," in data["model"] and llm_router is not None:
-            if (
-                data.get("fastest_response", None) is not None
-                and data["fastest_response"] == True
-            ):
-                tasks.append(llm_router.abatch_completion_fastest_response(**data))
-            else:
-                _models_csv_string = data.pop("model")
-                _models = [model.strip() for model in _models_csv_string.split(",")]
-                tasks.append(llm_router.abatch_completion(models=_models, **data))
-        elif "user_config" in data:
-            # initialize a new router instance. make request using this Router
-            router_config = data.pop("user_config")
-            user_router = litellm.Router(**router_config)
-            tasks.append(user_router.acompletion(**data))
-        elif (
-            llm_router is not None and data["model"] in router_model_names
-        ):  # model in router model list
-            tasks.append(llm_router.acompletion(**data))
-        elif (
-            llm_router is not None and data["model"] in llm_router.get_model_ids()
-        ):  # model in router model list
-            tasks.append(llm_router.acompletion(**data))
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data["model"] in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            tasks.append(llm_router.acompletion(**data))
-        elif (
-            llm_router is not None and data["model"] in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            tasks.append(llm_router.acompletion(**data, specific_deployment=True))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and llm_router.router_general_settings.pass_through_all_models is True
-        ):
-            tasks.append(litellm.acompletion(**data))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            tasks.append(llm_router.acompletion(**data))
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            tasks.append(litellm.acompletion(**data))
-        else:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "error": "chat_completion: Invalid model name passed in model="
-                    + data.get("model", "")
-                },
-            )
+        llm_call = await route_request(
+            data=data,
+            route_type="acompletion",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
+        tasks.append(llm_call)
 
         # wait for call to end
         llm_responses = asyncio.gather(
@@ -3290,58 +3236,15 @@ async def completion(
         )
 
         ### ROUTE THE REQUESTs ###
-        router_model_names = llm_router.model_names if llm_router is not None else []
-        # skip router if user passed their key
-        if "api_key" in data:
-            llm_response = asyncio.create_task(litellm.atext_completion(**data))
-        elif (
-            llm_router is not None and data["model"] in router_model_names
-        ):  # model in router model list
-            llm_response = asyncio.create_task(llm_router.atext_completion(**data))
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data["model"] in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            llm_response = asyncio.create_task(llm_router.atext_completion(**data))
-        elif (
-            llm_router is not None and data["model"] in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            llm_response = asyncio.create_task(
-                llm_router.atext_completion(**data, specific_deployment=True)
-            )
-        elif (
-            llm_router is not None and data["model"] in llm_router.get_model_ids()
-        ):  # model in router model list
-            llm_response = asyncio.create_task(llm_router.atext_completion(**data))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and llm_router.router_general_settings.pass_through_all_models is True
-        ):
-            llm_response = asyncio.create_task(litellm.atext_completion(**data))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            llm_response = asyncio.create_task(llm_router.atext_completion(**data))
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            llm_response = asyncio.create_task(litellm.atext_completion(**data))
-        else:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "error": "completion: Invalid model name passed in model="
-                    + data.get("model", "")
-                },
-            )
+        llm_call = await route_request(
+            data=data,
+            route_type="atext_completion",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
 
         # Await the llm_response task
-        response = await llm_response
+        response = await llm_call
 
         hidden_params = getattr(response, "_hidden_params", {}) or {}
         model_id = hidden_params.get("model_id", None) or ""
@@ -3555,59 +3458,13 @@ async def embeddings(
         )
 
         ## ROUTE TO CORRECT ENDPOINT ##
-        # skip router if user passed their key
-        if "api_key" in data:
-            tasks.append(litellm.aembedding(**data))
-        elif "user_config" in data:
-            # initialize a new router instance. make request using this Router
-            router_config = data.pop("user_config")
-            user_router = litellm.Router(**router_config)
-            tasks.append(user_router.aembedding(**data))
-        elif (
-            llm_router is not None and data["model"] in router_model_names
-        ):  # model in router model list
-            tasks.append(llm_router.aembedding(**data))
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data["model"] in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            tasks.append(
-                llm_router.aembedding(**data)
-            )  # ensure this goes the llm_router, router will do the correct alias mapping
-        elif (
-            llm_router is not None and data["model"] in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            tasks.append(llm_router.aembedding(**data, specific_deployment=True))
-        elif (
-            llm_router is not None and data["model"] in llm_router.get_model_ids()
-        ):  # model in router deployments, calling a specific deployment on the router
-            tasks.append(llm_router.aembedding(**data))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and llm_router.router_general_settings.pass_through_all_models is True
-        ):
-            tasks.append(litellm.aembedding(**data))
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            tasks.append(llm_router.aembedding(**data))
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            tasks.append(litellm.aembedding(**data))
-        else:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "error": "embeddings: Invalid model name passed in model="
-                    + data.get("model", "")
-                },
-            )
+        llm_call = await route_request(
+            data=data,
+            route_type="aembedding",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
+        tasks.append(llm_call)
 
         # wait for call to end
         llm_responses = asyncio.gather(
@@ -3738,46 +3595,13 @@ async def image_generation(
         )
 
         ## ROUTE TO CORRECT ENDPOINT ##
-        # skip router if user passed their key
-        if "api_key" in data:
-            response = await litellm.aimage_generation(**data)
-        elif (
-            llm_router is not None and data["model"] in router_model_names
-        ):  # model in router model list
-            response = await llm_router.aimage_generation(**data)
-        elif (
-            llm_router is not None and data["model"] in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.aimage_generation(
-                **data, specific_deployment=True
-            )
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data["model"] in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            response = await llm_router.aimage_generation(
-                **data
-            )  # ensure this goes the llm_router, router will do the correct alias mapping
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.aimage_generation(**data)
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            response = await litellm.aimage_generation(**data)
-        else:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "error": "image_generation: Invalid model name passed in model="
-                    + data.get("model", "")
-                },
-            )
+        llm_call = await route_request(
+            data=data,
+            route_type="aimage_generation",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
+        response = await llm_call
 
         ### ALERTING ###
         asyncio.create_task(
@@ -3885,44 +3709,13 @@ async def audio_speech(
         )
 
         ## ROUTE TO CORRECT ENDPOINT ##
-        # skip router if user passed their key
-        if "api_key" in data:
-            response = await litellm.aspeech(**data)
-        elif (
-            llm_router is not None and data["model"] in router_model_names
-        ):  # model in router model list
-            response = await llm_router.aspeech(**data)
-        elif (
-            llm_router is not None and data["model"] in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.aspeech(**data, specific_deployment=True)
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data["model"] in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            response = await llm_router.aspeech(
-                **data
-            )  # ensure this goes the llm_router, router will do the correct alias mapping
-        elif (
-            llm_router is not None
-            and data["model"] not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.aspeech(**data)
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            response = await litellm.aspeech(**data)
-        else:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST,
-                detail={
-                    "error": "audio_speech: Invalid model name passed in model="
-                    + data.get("model", "")
-                },
-            )
+        llm_call = await route_request(
+            data=data,
+            route_type="aspeech",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
+        response = await llm_call
 
         ### ALERTING ###
         asyncio.create_task(
@@ -4055,47 +3848,13 @@ async def audio_transcriptions(
             )
 
             ## ROUTE TO CORRECT ENDPOINT ##
-            # skip router if user passed their key
-            if "api_key" in data:
-                response = await litellm.atranscription(**data)
-            elif (
-                llm_router is not None and data["model"] in router_model_names
-            ):  # model in router model list
-                response = await llm_router.atranscription(**data)
-
-            elif (
-                llm_router is not None and data["model"] in llm_router.deployment_names
-            ):  # model in router deployments, calling a specific deployment on the router
-                response = await llm_router.atranscription(
-                    **data, specific_deployment=True
-                )
-            elif (
-                llm_router is not None
-                and llm_router.model_group_alias is not None
-                and data["model"] in llm_router.model_group_alias
-            ):  # model set in model_group_alias
-                response = await llm_router.atranscription(
-                    **data
-                )  # ensure this goes the llm_router, router will do the correct alias mapping
-            elif (
-                llm_router is not None
-                and data["model"] not in router_model_names
-                and (
-                    llm_router.default_deployment is not None
-                    or len(llm_router.provider_default_deployments) > 0
-                )
-            ):  # model in router deployments, calling a specific deployment on the router
-                response = await llm_router.atranscription(**data)
-            elif user_model is not None:  # `litellm --model <your-model-name>`
-                response = await litellm.atranscription(**data)
-            else:
-                raise HTTPException(
-                    status_code=status.HTTP_400_BAD_REQUEST,
-                    detail={
-                        "error": "audio_transcriptions: Invalid model name passed in model="
-                        + data.get("model", "")
-                    },
-                )
+            llm_call = await route_request(
+                data=data,
+                route_type="atranscription",
+                llm_router=llm_router,
+                user_model=user_model,
+            )
+            response = await llm_call
         except Exception as e:
             raise HTTPException(status_code=500, detail=str(e))
         finally:
@@ -5311,40 +5070,13 @@ async def moderations(
         start_time = time.time()
 
         ## ROUTE TO CORRECT ENDPOINT ##
-        # skip router if user passed their key
-        if "api_key" in data:
-            response = await litellm.amoderation(**data)
-        elif (
-            llm_router is not None and data.get("model") in router_model_names
-        ):  # model in router model list
-            response = await llm_router.amoderation(**data)
-        elif (
-            llm_router is not None and data.get("model") in llm_router.deployment_names
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.amoderation(**data, specific_deployment=True)
-        elif (
-            llm_router is not None
-            and llm_router.model_group_alias is not None
-            and data.get("model") in llm_router.model_group_alias
-        ):  # model set in model_group_alias
-            response = await llm_router.amoderation(
-                **data
-            )  # ensure this goes the llm_router, router will do the correct alias mapping
-        elif (
-            llm_router is not None
-            and data.get("model") not in router_model_names
-            and (
-                llm_router.default_deployment is not None
-                or len(llm_router.provider_default_deployments) > 0
-            )
-        ):  # model in router deployments, calling a specific deployment on the router
-            response = await llm_router.amoderation(**data)
-        elif user_model is not None:  # `litellm --model <your-model-name>`
-            response = await litellm.amoderation(**data)
-        else:
-            # /moderations does not need a "model" passed
-            # see https://platform.openai.com/docs/api-reference/moderations
-            response = await litellm.amoderation(**data)
+        llm_call = await route_request(
+            data=data,
+            route_type="amoderation",
+            llm_router=llm_router,
+            user_model=user_model,
+        )
+        response = await llm_call
 
         ### ALERTING ###
         asyncio.create_task(

litellm/proxy/route_llm_request.py

@@ -0,0 +1,117 @@
+from typing import TYPE_CHECKING, Any, Literal, Optional, Union
+
+from fastapi import (
+    Depends,
+    FastAPI,
+    File,
+    Form,
+    Header,
+    HTTPException,
+    Path,
+    Request,
+    Response,
+    UploadFile,
+    status,
+)
+
+import litellm
+from litellm._logging import verbose_logger
+
+if TYPE_CHECKING:
+    from litellm.router import Router as _Router
+
+    LitellmRouter = _Router
+else:
+    LitellmRouter = Any
+
+
+ROUTE_ENDPOINT_MAPPING = {
+    "acompletion": "/chat/completions",
+    "atext_completion": "/completions",
+    "aembedding": "/embeddings",
+    "aimage_generation": "/image/generations",
+    "aspeech": "/audio/speech",
+    "atranscription": "/audio/transcriptions",
+    "amoderation": "/moderations",
+}
+
+
+async def route_request(
+    data: dict,
+    llm_router: Optional[LitellmRouter],
+    user_model: Optional[str],
+    route_type: Literal[
+        "acompletion",
+        "atext_completion",
+        "aembedding",
+        "aimage_generation",
+        "aspeech",
+        "atranscription",
+        "amoderation",
+    ],
+):
+    """
+    Common helper to route the request
+
+    """
+    router_model_names = llm_router.model_names if llm_router is not None else []
+
+    if "api_key" in data:
+        return getattr(litellm, f"{route_type}")(**data)
+
+    elif "user_config" in data:
+        router_config = data.pop("user_config")
+        user_router = litellm.Router(**router_config)
+        return getattr(user_router, f"{route_type}")(**data)
+
+    elif (
+        route_type == "acompletion"
+        and data.get("model", "") is not None
+        and "," in data.get("model", "")
+        and llm_router is not None
+    ):
+        if data.get("fastest_response", False):
+            return llm_router.abatch_completion_fastest_response(**data)
+        else:
+            models = [model.strip() for model in data.pop("model").split(",")]
+            return llm_router.abatch_completion(models=models, **data)
+
+    elif llm_router is not None:
+        if (
+            data["model"] in router_model_names
+            or data["model"] in llm_router.get_model_ids()
+        ):
+            return getattr(llm_router, f"{route_type}")(**data)
+
+        elif (
+            llm_router.model_group_alias is not None
+            and data["model"] in llm_router.model_group_alias
+        ):
+            return getattr(llm_router, f"{route_type}")(**data)
+
+        elif data["model"] in llm_router.deployment_names:
+            return getattr(llm_router, f"{route_type}")(
+                **data, specific_deployment=True
+            )
+
+        elif data["model"] not in router_model_names:
+            if llm_router.router_general_settings.pass_through_all_models:
+                return getattr(litellm, f"{route_type}")(**data)
+            elif (
+                llm_router.default_deployment is not None
+                or len(llm_router.provider_default_deployments) > 0
+            ):
+                return getattr(llm_router, f"{route_type}")(**data)
+
+    elif user_model is not None:
+        return getattr(litellm, f"{route_type}")(**data)
+
+    # if no route found then it's a bad request
+    route_name = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
+    raise HTTPException(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        detail={
+            "error": f"{route_name}: Invalid model name passed in model="
+            + data.get("model", "")
+        },
+    )

litellm/tests/test_proxy_exception_mapping.py

@@ -229,8 +229,9 @@ def test_chat_completion_exception_any_model(client):
         )
         assert isinstance(openai_exception, openai.BadRequestError)
         _error_message = openai_exception.message
-        assert "chat_completion: Invalid model name passed in model=Lite-GPT-12" in str(
-            _error_message
+        assert (
+            "/chat/completions: Invalid model name passed in model=Lite-GPT-12"
+            in str(_error_message)
         )
 
     except Exception as e:
@@ -259,7 +260,7 @@ def test_embedding_exception_any_model(client):
         print("Exception raised=", openai_exception)
         assert isinstance(openai_exception, openai.BadRequestError)
         _error_message = openai_exception.message
-        assert "embeddings: Invalid model name passed in model=Lite-GPT-12" in str(
+        assert "/embeddings: Invalid model name passed in model=Lite-GPT-12" in str(
             _error_message
         )