From 596c71c36062212afa0317bd4fb779f845aa2146 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Wed, 30 Apr 2025 21:57:06 -0700 Subject: [PATCH] Add low-level HTTP client (#10452) The client provides access to a low-level HTTP client for making direct requests to the LiteLLM proxy server. This is useful when you need more control or when working with endpoints that don't yet have a high-level interface. ```python In [2]: client.http.request( ...: method="POST", ...: uri="/health/test_connection", ...: json={ ...: "litellm_params": { ...: "model": "gpt-4", ...: "custom_llm_provider": "azure_ai", ...: "litellm_credential_name": None, ...: "api_key": "6xxxxxxx", ...: "api_base": "https://litellm8397336933...", ...: }, ...: "mode": "chat", ...: }, ...: ) Out[2]: {'status': 'error', 'result': {'model': 'gpt-4', 'custom_llm_provider': 'azure_ai', 'litellm_credential_name': None, 'api_base': 'https://litellm8397336933...', ... ``` --- litellm/proxy/client/README.md | 49 +++++ litellm/proxy/client/client.py | 14 +- litellm/proxy/client/http_client.py | 95 ++++++++++ poetry.lock | 24 ++- pyproject.toml | 1 + tests/litellm/proxy/client/test_client.py | 40 +++++ .../litellm/proxy/client/test_http_client.py | 167 ++++++++++++++++++ 7 files changed, 386 insertions(+), 4 deletions(-) create mode 100644 litellm/proxy/client/http_client.py create mode 100644 tests/litellm/proxy/client/test_http_client.py diff --git a/litellm/proxy/client/README.md b/litellm/proxy/client/README.md index 5f09817aaa..c83944288f 100644 --- a/litellm/proxy/client/README.md +++ b/litellm/proxy/client/README.md @@ -157,6 +157,55 @@ groups = client.model_groups.list() client.model_groups.delete(name="gpt4-group") ``` +## Low-Level HTTP Client + +The client provides access to a low-level HTTP client for making direct requests +to the LiteLLM proxy server. This is useful when you need more control or when +working with endpoints that don't yet have a high-level interface. + +```python +# Access the HTTP client +client = Client( + base_url="http://localhost:4000", + api_key="sk-api-key" +) + +# Make a custom request +response = client.http.request( + method="POST", + uri="/health/test_connection", + json={ + "litellm_params": { + "model": "gpt-4", + "api_key": "your-api-key", + "api_base": "https://api.openai.com/v1" + }, + "mode": "chat" + } +) + +# The response is automatically parsed from JSON +print(response) +``` + +### HTTP Client Features + +- Automatic URL handling (handles trailing/leading slashes) +- Built-in authentication (adds Bearer token if `api_key` is provided) +- JSON request/response handling +- Configurable timeout (default: 30 seconds) +- Comprehensive error handling +- Support for custom headers and request parameters + +### HTTP Client `request` method parameters + +- `method`: HTTP method (GET, POST, PUT, DELETE, etc.) +- `uri`: URI path (will be appended to base_url) +- `data`: (optional) Data to send in the request body +- `json`: (optional) JSON data to send in the request body +- `headers`: (optional) Custom HTTP headers +- Additional keyword arguments are passed to the underlying requests library + ## Error Handling The client provides clear error handling with custom exceptions: diff --git a/litellm/proxy/client/client.py b/litellm/proxy/client/client.py index b4d0d77ffe..93e877d156 100644 --- a/litellm/proxy/client/client.py +++ b/litellm/proxy/client/client.py @@ -1,4 +1,6 @@ from typing import Optional + +from .http_client import HTTPClient from .models import ModelsManagementClient from .model_groups import ModelGroupsManagementClient from .chat import ChatClient @@ -9,18 +11,26 @@ from .credentials import CredentialsManagementClient class Client: """Main client for interacting with the LiteLLM proxy API.""" - def __init__(self, base_url: str, api_key: Optional[str] = None): + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + timeout: int = 30, + ): """ Initialize the LiteLLM proxy client. Args: - base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000") + base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:4000") api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token. + timeout: Request timeout in seconds (default: 30) """ self._base_url = base_url.rstrip("/") # Remove trailing slash if present self._api_key = api_key # Initialize resource clients + + self.http = HTTPClient(base_url=base_url, api_key=api_key, timeout=timeout) self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key) self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key) self.chat = ChatClient(base_url=self._base_url, api_key=self._api_key) diff --git a/litellm/proxy/client/http_client.py b/litellm/proxy/client/http_client.py new file mode 100644 index 0000000000..4357f6e35b --- /dev/null +++ b/litellm/proxy/client/http_client.py @@ -0,0 +1,95 @@ +"""HTTP client for making requests to the LiteLLM proxy server.""" + +from typing import Any, Dict, Optional, Union +import requests + + +class HTTPClient: + """HTTP client for making requests to the LiteLLM proxy server.""" + + def __init__(self, base_url: str, api_key: Optional[str] = None, timeout: int = 30): + """Initialize the HTTP client. + + Args: + base_url: Base URL of the LiteLLM proxy server + api_key: Optional API key for authentication + timeout: Request timeout in seconds (default: 30) + """ + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._timeout = timeout + + def request( + self, + method: str, + uri: str, + *, + data: Optional[Union[Dict[str, Any], list, bytes]] = None, + json: Optional[Union[Dict[str, Any], list]] = None, + headers: Optional[Dict[str, str]] = None, + **kwargs: Any, + ) -> Any: + """Make an HTTP request to the LiteLLM proxy server. + + This method is used to make generic requests to the LiteLLM proxy + server, when there is not a specific client or method for the request. + + Args: + method: HTTP method (GET, POST, PUT, DELETE, etc.) + uri: URI path (will be appended to base_url) (e.g., "/credentials") + data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the request. + json: (optional) A JSON serializable Python object to send in the body + of the request. + headers: (optional) Dictionary of HTTP headers to send with the request. + **kwargs: Additional keyword arguments to pass to the request. + + Returns: + Parsed JSON response from the server + + Raises: + requests.exceptions.RequestException: If the request fails + ValueError: If the response is not valid JSON + + Example: + >>> client.http.request("POST", "/health/test_connection", json={ + "litellm_params": { + "model": "gpt-4", + "custom_llm_provider": "azure_ai", + "litellm_credential_name": None, + "api_key": "6xxxxxxx", + "api_base": "https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21", + }, + "mode": "chat", + }) + {'status': 'error', + 'result': {'model': 'gpt-4', + 'custom_llm_provider': 'azure_ai', + 'litellm_credential_name': None, + ... + """ + # Build complete URL + url = f"{self._base_url}/{uri.lstrip('/')}" + + # Prepare headers + request_headers = {} + if headers: + request_headers.update(headers) + if self._api_key: + request_headers["Authorization"] = f"Bearer {self._api_key}" + + response = requests.request( + method=method, + url=url, + data=data, + json=json, + headers=request_headers, + timeout=self._timeout, + **kwargs, + ) + + # Raise for HTTP errors + response.raise_for_status() + + # Parse and return JSON response + return response.json() diff --git a/poetry.lock b/poetry.lock index bb0f00d571..b72519b8d8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3235,7 +3235,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -3524,6 +3524,26 @@ files = [ [package.dependencies] requests = "2.31.0" +[[package]] +name = "responses" +version = "0.25.7" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "responses-0.25.7-py3-none-any.whl", hash = "sha256:92ca17416c90fe6b35921f52179bff29332076bb32694c0df02dcac2c6bc043c"}, + {file = "responses-0.25.7.tar.gz", hash = "sha256:8ebae11405d7a5df79ab6fd54277f6f2bc29b2d002d0dd2d5c632594d1ddcedb"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "respx" version = "0.22.0" @@ -4623,4 +4643,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi", [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "ba7691ffdf305d212bf661fbe0f2a6919043852d993721189c6b7b92d4a127c0" +content-hash = "db62f54bc947ac8c8784b89744aa3f531eb36b798f9f3a8325375d65f4eae190" diff --git a/pyproject.toml b/pyproject.toml index acb418738d..d603c22bae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ pytest = "^7.4.3" pytest-mock = "^3.12.0" pytest-asyncio = "^0.21.1" requests-mock = "^1.12.1" +responses = "^0.25.7" respx = "^0.22.0" ruff = "^0.1.0" types-requests = "*" diff --git a/tests/litellm/proxy/client/test_client.py b/tests/litellm/proxy/client/test_client.py index 4a7f0fb93a..ceb872f87b 100644 --- a/tests/litellm/proxy/client/test_client.py +++ b/tests/litellm/proxy/client/test_client.py @@ -1,6 +1,7 @@ import pytest from litellm.proxy.client import Client, ModelsManagementClient, ChatClient from litellm.proxy.client.keys import KeysManagementClient +from litellm.proxy.client.http_client import HTTPClient @pytest.fixture @@ -36,6 +37,11 @@ def test_client_initialization(base_url, api_key): assert client.keys._base_url == base_url assert client.keys._api_key == api_key + # Check http client + assert isinstance(client.http, HTTPClient) + assert client.http._base_url == base_url + assert client.http._api_key == api_key + def test_client_initialization_strips_trailing_slash(): """Test that the client properly strips trailing slashes from base_url during initialization""" @@ -46,6 +52,7 @@ def test_client_initialization_strips_trailing_slash(): assert client.models._base_url == "http://localhost:8000" assert client.chat._base_url == "http://localhost:8000" assert client.keys._base_url == "http://localhost:8000" + assert client.http._base_url == "http://localhost:8000" def test_client_without_api_key(base_url): @@ -56,3 +63,36 @@ def test_client_without_api_key(base_url): assert client.models._api_key is None assert client.chat._api_key is None assert client.keys._api_key is None + assert client.http._api_key is None + + +def test_client_initialization(): + """Test that the client is initialized correctly.""" + client = Client( + base_url="http://localhost:4000", + api_key="test-key", + timeout=60, + ) + + # Check that http client is initialized correctly + assert isinstance(client.http, HTTPClient) + assert client.http._base_url == "http://localhost:4000" + assert client.http._api_key == "test-key" + assert client.http._timeout == 60 + + +def test_client_default_timeout(): + """Test that the client uses default timeout.""" + client = Client( + base_url="http://localhost:4000", + api_key="test-key", + ) + + assert client.http._timeout == 30 + + +def test_client_without_api_key(): + """Test that the client works without an API key.""" + client = Client(base_url="http://localhost:4000") + + assert client.http._api_key is None diff --git a/tests/litellm/proxy/client/test_http_client.py b/tests/litellm/proxy/client/test_http_client.py new file mode 100644 index 0000000000..4e7c16a537 --- /dev/null +++ b/tests/litellm/proxy/client/test_http_client.py @@ -0,0 +1,167 @@ +"""Tests for the HTTP client.""" + +import json +import pytest +import requests +import responses +from litellm.proxy.client.http_client import HTTPClient + + +@pytest.fixture +def client(): + """Create a test HTTP client.""" + return HTTPClient( + base_url="http://localhost:4000", + api_key="test-key", + ) + + +@responses.activate +def test_request_get(client): + """Test making a GET request.""" + # Mock response + responses.add( + responses.GET, + "http://localhost:4000/models", + json={"models": []}, + status=200, + ) + + # Make request + response = client.request("GET", "/models") + + # Check response + assert response == {"models": []} + + # Check request + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == "http://localhost:4000/models" + assert responses.calls[0].request.headers["Authorization"] == "Bearer test-key" + + +@responses.activate +def test_request_post_with_json(client): + """Test making a POST request with JSON data.""" + # Mock response + responses.add( + responses.POST, + "http://localhost:4000/models", + json={"id": "model-123"}, + status=200, + ) + + # Test data + json_data = { + "model": "gpt-4", + "params": {"temperature": 0.7} + } + + # Make request + response = client.request( + "POST", + "/models", + json=json_data, + ) + + # Check response + assert response == {"id": "model-123"} + + # Check request + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == "http://localhost:4000/models" + assert json.loads(responses.calls[0].request.body) == json_data + + +@responses.activate +def test_request_with_custom_headers(client): + """Test making a request with custom headers.""" + # Mock response + responses.add( + responses.GET, + "http://localhost:4000/models", + json={"models": []}, + status=200, + ) + + # Make request with custom headers + custom_headers = { + "X-Custom-Header": "test-value", + "Accept": "application/json", + } + response = client.request( + "GET", + "/models", + headers=custom_headers, + ) + + # Check request headers + assert len(responses.calls) == 1 + request_headers = responses.calls[0].request.headers + assert request_headers["X-Custom-Header"] == "test-value" + assert request_headers["Accept"] == "application/json" + assert request_headers["Authorization"] == "Bearer test-key" + + +@responses.activate +def test_request_http_error(client): + """Test handling of HTTP errors.""" + # Mock error response + responses.add( + responses.GET, + "http://localhost:4000/models", + json={"error": "Not authorized"}, + status=401, + ) + + # Check that request raises exception + with pytest.raises(requests.exceptions.HTTPError) as exc_info: + client.request("GET", "/models") + + assert exc_info.value.response.status_code == 401 + + +@responses.activate +def test_request_invalid_json(client): + """Test handling of invalid JSON responses.""" + # Mock invalid JSON response + responses.add( + responses.GET, + "http://localhost:4000/models", + body="not json", + status=200, + ) + + # Check that request raises exception + with pytest.raises(json.JSONDecodeError) as exc_info: + client.request("GET", "/models") + + +def test_base_url_trailing_slash(): + """Test that trailing slashes in base_url are handled correctly.""" + client = HTTPClient( + base_url="http://localhost:4000/", + api_key="test-key", + ) + assert client._base_url == "http://localhost:4000" + + +def test_uri_leading_slash(): + """Test that URIs with and without leading slashes work.""" + client = HTTPClient(base_url="http://localhost:4000") + + with responses.RequestsMock() as rsps: + # Mock endpoint + rsps.add( + responses.GET, + "http://localhost:4000/models", + json={"models": []}, + ) + + # Both of these should work and hit the same endpoint + client.request("GET", "/models") + client.request("GET", "models") + + # Check that both requests went to the same URL + assert len(rsps.calls) == 2 + assert rsps.calls[0].request.url == "http://localhost:4000/models" + assert rsps.calls[1].request.url == "http://localhost:4000/models" \ No newline at end of file