tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-24 00:28:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add low-level HTTP client (#10452)

Browse Source 
The client provides access to a low-level HTTP client for making direct
requests to the LiteLLM proxy server. This is useful when you need more
control or when working with endpoints that don't yet have a high-level
interface.

```python
In [2]: client.http.request(
   ...:     method="POST",
   ...:     uri="/health/test_connection",
   ...:     json={
   ...:         "litellm_params": {
   ...:             "model": "gpt-4",
   ...:             "custom_llm_provider": "azure_ai",
   ...:             "litellm_credential_name": None,
   ...:             "api_key": "6xxxxxxx",
   ...:             "api_base": "https://litellm8397336933...",
   ...:         },
   ...:         "mode": "chat",
   ...:     },
   ...: )
Out[2]:
{'status': 'error',
 'result': {'model': 'gpt-4',
  'custom_llm_provider': 'azure_ai',
  'litellm_credential_name': None,
  'api_base': 'https://litellm8397336933...',
  ...
```
This commit is contained in:
Marc Abramowitz
2025-04-30 21:57:06 -07:00
committed by GitHub
parent be885e4a17
commit 596c71c360
7 changed files with 386 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
litellm/proxy/client/README.md
+49
View File
@@ -157,6 +157,55 @@ groups = client.model_groups.list()
client.model_groups.delete(name="gpt4-group")
```
## Low-Level HTTP Client
The client provides access to a low-level HTTP client for making direct requests
to the LiteLLM proxy server. This is useful when you need more control or when
working with endpoints that don't yet have a high-level interface.
```python
# Access the HTTP client
client = Client(
base_url="http://localhost:4000",
api_key="sk-api-key"
)
# Make a custom request
response = client.http.request(
method="POST",
uri="/health/test_connection",
json={
"litellm_params": {
"model": "gpt-4",
"api_key": "your-api-key",
"api_base": "https://api.openai.com/v1"
},
"mode": "chat"
}
)
# The response is automatically parsed from JSON
print(response)
```
### HTTP Client Features
- Automatic URL handling (handles trailing/leading slashes)
- Built-in authentication (adds Bearer token if `api_key` is provided)
- JSON request/response handling
- Configurable timeout (default: 30 seconds)
- Comprehensive error handling
- Support for custom headers and request parameters
### HTTP Client `request` method parameters
- `method`: HTTP method (GET, POST, PUT, DELETE, etc.)
- `uri`: URI path (will be appended to base_url)
- `data`: (optional) Data to send in the request body
- `json`: (optional) JSON data to send in the request body
- `headers`: (optional) Custom HTTP headers
- Additional keyword arguments are passed to the underlying requests library
## Error Handling
The client provides clear error handling with custom exceptions:
litellm/proxy/client/client.py
+12 -2
View File
@@ -1,4 +1,6 @@
from typing import Optional
from .http_client import HTTPClient
from .models import ModelsManagementClient
from .model_groups import ModelGroupsManagementClient
from .chat import ChatClient
@@ -9,18 +11,26 @@ from .credentials import CredentialsManagementClient
class Client:
"""Main client for interacting with the LiteLLM proxy API."""
def __init__(self, base_url: str, api_key: Optional[str] = None):
def __init__(
self,
base_url: str,
api_key: Optional[str] = None,
timeout: int = 30,
):
"""
Initialize the LiteLLM proxy client.
Args:
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:8000")
base_url (str): The base URL of the LiteLLM proxy server (e.g., "http://localhost:4000")
api_key (Optional[str]): API key for authentication. If provided, it will be sent as a Bearer token.
timeout: Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
self._api_key = api_key
# Initialize resource clients
self.http = HTTPClient(base_url=base_url, api_key=api_key, timeout=timeout)
self.models = ModelsManagementClient(base_url=self._base_url, api_key=self._api_key)
self.model_groups = ModelGroupsManagementClient(base_url=self._base_url, api_key=self._api_key)
self.chat = ChatClient(base_url=self._base_url, api_key=self._api_key)
litellm/proxy/client/http_client.py
+95
View File
@@ -0,0 +1,95 @@
"""HTTP client for making requests to the LiteLLM proxy server."""
from typing import Any, Dict, Optional, Union
import requests
class HTTPClient:
"""HTTP client for making requests to the LiteLLM proxy server."""
def __init__(self, base_url: str, api_key: Optional[str] = None, timeout: int = 30):
"""Initialize the HTTP client.
Args:
base_url: Base URL of the LiteLLM proxy server
api_key: Optional API key for authentication
timeout: Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/")
self._api_key = api_key
self._timeout = timeout
def request(
self,
method: str,
uri: str,
*,
data: Optional[Union[Dict[str, Any], list, bytes]] = None,
json: Optional[Union[Dict[str, Any], list]] = None,
headers: Optional[Dict[str, str]] = None,
**kwargs: Any,
) -> Any:
"""Make an HTTP request to the LiteLLM proxy server.
This method is used to make generic requests to the LiteLLM proxy
server, when there is not a specific client or method for the request.
Args:
method: HTTP method (GET, POST, PUT, DELETE, etc.)
uri: URI path (will be appended to base_url) (e.g., "/credentials")
data: (optional) Dictionary, list of tuples, bytes, or file-like
object to send in the body of the request.
json: (optional) A JSON serializable Python object to send in the body
of the request.
headers: (optional) Dictionary of HTTP headers to send with the request.
**kwargs: Additional keyword arguments to pass to the request.
Returns:
Parsed JSON response from the server
Raises:
requests.exceptions.RequestException: If the request fails
ValueError: If the response is not valid JSON
Example:
>>> client.http.request("POST", "/health/test_connection", json={
"litellm_params": {
"model": "gpt-4",
"custom_llm_provider": "azure_ai",
"litellm_credential_name": None,
"api_key": "6xxxxxxx",
"api_base": "https://litellm8397336933.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21",
},
"mode": "chat",
})
{'status': 'error',
'result': {'model': 'gpt-4',
'custom_llm_provider': 'azure_ai',
'litellm_credential_name': None,
...
"""
# Build complete URL
url = f"{self._base_url}/{uri.lstrip('/')}"
# Prepare headers
request_headers = {}
if headers:
request_headers.update(headers)
if self._api_key:
request_headers["Authorization"] = f"Bearer {self._api_key}"
response = requests.request(
method=method,
url=url,
data=data,
json=json,
headers=request_headers,
timeout=self._timeout,
**kwargs,
)
# Raise for HTTP errors
response.raise_for_status()
# Parse and return JSON response
return response.json()
poetry.lock
Generated
+22 -2
View File
@@ -3235,7 +3235,7 @@ version = "6.0.2"
description = "YAML parser and emitter for Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
groups = ["main", "dev"]
files = [
{file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
{file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -3524,6 +3524,26 @@ files = [
[package.dependencies]
requests = "2.31.0"
[[package]]
name = "responses"
version = "0.25.7"
description = "A utility library for mocking out the `requests` Python library."
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "responses-0.25.7-py3-none-any.whl", hash = "sha256:92ca17416c90fe6b35921f52179bff29332076bb32694c0df02dcac2c6bc043c"},
{file = "responses-0.25.7.tar.gz", hash = "sha256:8ebae11405d7a5df79ab6fd54277f6f2bc29b2d002d0dd2d5c632594d1ddcedb"},
]
[package.dependencies]
pyyaml = "*"
requests = ">=2.30.0,<3.0"
urllib3 = ">=1.25.10,<3.0"
[package.extras]
tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"]
[[package]]
name = "respx"
version = "0.22.0"
@@ -4623,4 +4643,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi",
[metadata]
lock-version = "2.1"
python-versions = ">=3.8.1,<4.0, !=3.9.7"
content-hash = "ba7691ffdf305d212bf661fbe0f2a6919043852d993721189c6b7b92d4a127c0"
content-hash = "db62f54bc947ac8c8784b89744aa3f531eb36b798f9f3a8325375d65f4eae190"
pyproject.toml
+1
View File
@@ -102,6 +102,7 @@ pytest = "^7.4.3"
pytest-mock = "^3.12.0"
pytest-asyncio = "^0.21.1"
requests-mock = "^1.12.1"
responses = "^0.25.7"
respx = "^0.22.0"
ruff = "^0.1.0"
types-requests = "*"
tests/litellm/proxy/client/test_client.py
+40
View File
@@ -1,6 +1,7 @@
import pytest
from litellm.proxy.client import Client, ModelsManagementClient, ChatClient
from litellm.proxy.client.keys import KeysManagementClient
from litellm.proxy.client.http_client import HTTPClient
@pytest.fixture
@@ -36,6 +37,11 @@ def test_client_initialization(base_url, api_key):
assert client.keys._base_url == base_url
assert client.keys._api_key == api_key
# Check http client
assert isinstance(client.http, HTTPClient)
assert client.http._base_url == base_url
assert client.http._api_key == api_key
def test_client_initialization_strips_trailing_slash():
"""Test that the client properly strips trailing slashes from base_url during initialization"""
@@ -46,6 +52,7 @@ def test_client_initialization_strips_trailing_slash():
assert client.models._base_url == "http://localhost:8000"
assert client.chat._base_url == "http://localhost:8000"
assert client.keys._base_url == "http://localhost:8000"
assert client.http._base_url == "http://localhost:8000"
def test_client_without_api_key(base_url):
@@ -56,3 +63,36 @@ def test_client_without_api_key(base_url):
assert client.models._api_key is None
assert client.chat._api_key is None
assert client.keys._api_key is None
assert client.http._api_key is None
def test_client_initialization():
"""Test that the client is initialized correctly."""
client = Client(
base_url="http://localhost:4000",
api_key="test-key",
timeout=60,
)
# Check that http client is initialized correctly
assert isinstance(client.http, HTTPClient)
assert client.http._base_url == "http://localhost:4000"
assert client.http._api_key == "test-key"
assert client.http._timeout == 60
def test_client_default_timeout():
"""Test that the client uses default timeout."""
client = Client(
base_url="http://localhost:4000",
api_key="test-key",
)
assert client.http._timeout == 30
def test_client_without_api_key():
"""Test that the client works without an API key."""
client = Client(base_url="http://localhost:4000")
assert client.http._api_key is None
tests/litellm/proxy/client/test_http_client.py
+167
View File
@@ -0,0 +1,167 @@
"""Tests for the HTTP client."""
import json
import pytest
import requests
import responses
from litellm.proxy.client.http_client import HTTPClient
@pytest.fixture
def client():
"""Create a test HTTP client."""
return HTTPClient(
base_url="http://localhost:4000",
api_key="test-key",
)
@responses.activate
def test_request_get(client):
"""Test making a GET request."""
# Mock response
responses.add(
responses.GET,
"http://localhost:4000/models",
json={"models": []},
status=200,
)
# Make request
response = client.request("GET", "/models")
# Check response
assert response == {"models": []}
# Check request
assert len(responses.calls) == 1
assert responses.calls[0].request.url == "http://localhost:4000/models"
assert responses.calls[0].request.headers["Authorization"] == "Bearer test-key"
@responses.activate
def test_request_post_with_json(client):
"""Test making a POST request with JSON data."""
# Mock response
responses.add(
responses.POST,
"http://localhost:4000/models",
json={"id": "model-123"},
status=200,
)
# Test data
json_data = {
"model": "gpt-4",
"params": {"temperature": 0.7}
}
# Make request
response = client.request(
"POST",
"/models",
json=json_data,
)
# Check response
assert response == {"id": "model-123"}
# Check request
assert len(responses.calls) == 1
assert responses.calls[0].request.url == "http://localhost:4000/models"
assert json.loads(responses.calls[0].request.body) == json_data
@responses.activate
def test_request_with_custom_headers(client):
"""Test making a request with custom headers."""
# Mock response
responses.add(
responses.GET,
"http://localhost:4000/models",
json={"models": []},
status=200,
)
# Make request with custom headers
custom_headers = {
"X-Custom-Header": "test-value",
"Accept": "application/json",
}
response = client.request(
"GET",
"/models",
headers=custom_headers,
)
# Check request headers
assert len(responses.calls) == 1
request_headers = responses.calls[0].request.headers
assert request_headers["X-Custom-Header"] == "test-value"
assert request_headers["Accept"] == "application/json"
assert request_headers["Authorization"] == "Bearer test-key"
@responses.activate
def test_request_http_error(client):
"""Test handling of HTTP errors."""
# Mock error response
responses.add(
responses.GET,
"http://localhost:4000/models",
json={"error": "Not authorized"},
status=401,
)
# Check that request raises exception
with pytest.raises(requests.exceptions.HTTPError) as exc_info:
client.request("GET", "/models")
assert exc_info.value.response.status_code == 401
@responses.activate
def test_request_invalid_json(client):
"""Test handling of invalid JSON responses."""
# Mock invalid JSON response
responses.add(
responses.GET,
"http://localhost:4000/models",
body="not json",
status=200,
)
# Check that request raises exception
with pytest.raises(json.JSONDecodeError) as exc_info:
client.request("GET", "/models")
def test_base_url_trailing_slash():
"""Test that trailing slashes in base_url are handled correctly."""
client = HTTPClient(
base_url="http://localhost:4000/",
api_key="test-key",
)
assert client._base_url == "http://localhost:4000"
def test_uri_leading_slash():
"""Test that URIs with and without leading slashes work."""
client = HTTPClient(base_url="http://localhost:4000")
with responses.RequestsMock() as rsps:
# Mock endpoint
rsps.add(
responses.GET,
"http://localhost:4000/models",
json={"models": []},
)
# Both of these should work and hit the same endpoint
client.request("GET", "/models")
client.request("GET", "models")
# Check that both requests went to the same URL
assert len(rsps.calls) == 2
assert rsps.calls[0].request.url == "http://localhost:4000/models"
assert rsps.calls[1].request.url == "http://localhost:4000/models"