diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000000..7b4f15e8c8 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,262 @@ +# LiteLLM Architecture - LiteLLM SDK + AI Gateway + +This document helps contributors understand where to make changes in LiteLLM. + +--- + +## 1. AI Gateway (Proxy) Request Flow + +The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features. + +```mermaid +sequenceDiagram + participant Client + participant ProxyServer as proxy/proxy_server.py + participant Auth as proxy/auth/user_api_key_auth.py + participant Hooks as proxy/hooks/ + participant Router as router.py + participant Main as main.py + participant Handler as llms/custom_httpx/llm_http_handler.py + participant Transform as llms/{provider}/chat/transformation.py + participant Provider as LLM Provider API + + Client->>ProxyServer: POST /v1/chat/completions + ProxyServer->>Auth: user_api_key_auth() + ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter + ProxyServer->>Router: route_request() + Router->>Main: litellm.acompletion() + Main->>Handler: BaseLLMHTTPHandler.completion() + Handler->>Transform: ProviderConfig.transform_request() + Handler->>Provider: HTTP Request + Provider-->>Handler: Response + Handler->>Transform: ProviderConfig.transform_response() + Handler-->>Hooks: async_log_success_event() + Handler-->>Client: ModelResponse +``` + +### Proxy Components + +```mermaid +graph TD + subgraph "Incoming Request" + Client["POST /v1/chat/completions"] + end + + subgraph "proxy/proxy_server.py" + Endpoint["chat_completion()"] + end + + subgraph "proxy/auth/" + Auth["user_api_key_auth()"] + end + + subgraph "proxy/" + PreCall["litellm_pre_call_utils.py"] + RouteRequest["route_llm_request.py"] + end + + subgraph "litellm/" + Router["router.py"] + Main["main.py"] + end + + Client --> Endpoint + Endpoint --> Auth + Auth --> PreCall + PreCall --> RouteRequest + RouteRequest --> Router + Router --> Main + Main --> Client +``` + +**Key proxy files:** +- `proxy/proxy_server.py` - Main API endpoints +- `proxy/auth/` - Authentication (API keys, JWT, OAuth2) +- `proxy/hooks/` - Proxy-level callbacks +- `router.py` - Load balancing, fallbacks +- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.) + +**LLM-specific proxy endpoints:** + +| Endpoint | Directory | Purpose | +|----------|-----------|---------| +| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API | +| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough | +| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough | +| `/v1/images/*` | `proxy/image_endpoints/` | Image generation | +| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing | +| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads | +| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs | +| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking | +| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API | +| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores | +| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough | + +**Proxy Hooks** (`proxy/hooks/__init__.py`): + +| Hook | File | Purpose | +|------|------|---------| +| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits | +| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user | +| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation | +| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation | +| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection | + +To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`. + +--- + +## 2. SDK Request Flow + +The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway. + +```mermaid +graph TD + subgraph "SDK Entry Points" + Completion["litellm.completion()"] + Messages["litellm.messages()"] + end + + subgraph "main.py" + Main["completion()
acompletion()"] + end + + subgraph "utils.py" + GetProvider["get_llm_provider()"] + end + + subgraph "llms/custom_httpx/" + Handler["llm_http_handler.py
BaseLLMHTTPHandler"] + HTTP["http_handler.py
HTTPHandler / AsyncHTTPHandler"] + end + + subgraph "llms/{provider}/chat/" + TransformReq["transform_request()"] + TransformResp["transform_response()"] + end + + subgraph "litellm_core_utils/" + Streaming["streaming_handler.py"] + end + + subgraph "integrations/ (async, off main thread)" + Callbacks["custom_logger.py
Langfuse, Datadog, etc."] + end + + Completion --> Main + Messages --> Main + Main --> GetProvider + GetProvider --> Handler + Handler --> TransformReq + TransformReq --> HTTP + HTTP --> Provider["LLM Provider API"] + Provider --> HTTP + HTTP --> TransformResp + TransformResp --> Streaming + Streaming --> Response["ModelResponse"] + Response -.->|async| Callbacks +``` + +**Key SDK files:** +- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()` +- `utils.py` - `get_llm_provider()` resolves model → provider +- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator +- `llms/custom_httpx/http_handler.py` - Low-level HTTP client +- `llms/{provider}/chat/transformation.py` - Provider-specific transformations +- `litellm_core_utils/streaming_handler.py` - Streaming response handling +- `integrations/` - Async callbacks (Langfuse, Datadog, etc.) + +--- + +## 3. Translation Layer + +When a request comes in, it goes through a **translation layer** that converts between API formats. +Each translation is isolated in its own file, making it easy to test and modify independently. + +### Where to find translations + +| Incoming API | Provider | Translation File | +|--------------|----------|------------------| +| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` | +| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` | +| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` | +| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` | +| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` | +| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` | +| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` | +| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` | +| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` | +| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` | + +### Example: Debugging prompt caching + +If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works: + +1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py` +2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` +3. Compare how each handles `cache_control` in `transform_request()` + +### How translations work + +Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`): + +```python +class ProviderConfig(BaseConfig): + def transform_request(self, model, messages, optional_params, litellm_params, headers): + # Convert OpenAI format → Provider format + return {"messages": transformed_messages, ...} + + def transform_response(self, model, raw_response, model_response, logging_obj, ...): + # Convert Provider format → OpenAI format + return ModelResponse(choices=[...], usage=Usage(...)) +``` + +The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself. + +--- + +## 4. Adding/Modifying Providers + +### To add a new provider: + +1. Create `llms/{provider}/chat/transformation.py` +2. Implement `Config` class with `transform_request()` and `transform_response()` +3. Add tests in `tests/llm_translation/test_{provider}.py` + +### To add a feature (e.g., prompt caching): + +1. Find the translation file from the table above +2. Modify `transform_request()` to handle the new parameter +3. Add unit tests that verify the transformation + +### Testing checklist + +When adding a feature, verify it works across all paths: + +| Test | File Pattern | +|------|--------------| +| OpenAI passthrough | `tests/llm_translation/test_openai*.py` | +| Anthropic direct | `tests/llm_translation/test_anthropic*.py` | +| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` | +| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` | +| Vertex AI | `tests/llm_translation/test_vertex*.py` | +| Gemini | `tests/llm_translation/test_gemini*.py` | + +### Unit testing translations + +Translations are designed to be unit testable without making API calls: + +```python +from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig + +def test_prompt_caching_transform(): + config = BedrockConverseConfig() + result = config.transform_request( + model="anthropic.claude-3-opus", + messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}], + optional_params={}, + litellm_params={}, + headers={} + ) + assert "cachePoint" in str(result) # Verify cache_control was translated +``` diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 91cb1b8d27..39008d802b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1644,7 +1644,7 @@ "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -1944,7 +1944,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -2893,7 +2893,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -2964,7 +2964,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -2997,7 +2997,7 @@ "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3092,9 +3092,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "source": "https://azure.microsoft.com/en-us/blog/gpt-5-in-azure-ai-foundry-the-future-of-ai-apps-and-agents-starts-here/", @@ -3377,7 +3377,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -3440,7 +3440,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3500,7 +3500,7 @@ "cache_read_input_token_cost": 1.75e-07, "input_cost_per_token": 1.75e-06, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -3637,7 +3637,7 @@ "azure/gpt-5.2-pro": { "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -3668,7 +3668,7 @@ "azure/gpt-5.2-pro-2025-12-11": { "input_cost_per_token": 2.1e-05, "litellm_provider": "azure", - "max_input_tokens": 400000, + "max_input_tokens": 272000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -4684,7 +4684,7 @@ "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.38e-06, "litellm_provider": "azure", - "max_input_tokens": 272000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -7813,14 +7813,14 @@ "supports_vision": true }, "deepseek-chat": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -7834,14 +7834,14 @@ "supports_tool_choice": true }, "deepseek-reasoner": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, "litellm_provider": "deepseek", "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.2e-07, "source": "https://api-docs.deepseek.com/quick_start/pricing", "supported_endpoints": [ "/v1/chat/completions" @@ -10045,15 +10045,15 @@ }, "deepseek/deepseek-chat": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 7e-08, - "input_cost_per_token": 2.7e-07, - "input_cost_per_token_cache_hit": 7e-08, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", - "max_input_tokens": 65536, + "max_input_tokens": 128000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.1e-06, + "output_cost_per_token": 4.2e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -10089,14 +10089,15 @@ "supports_tool_choice": true }, "deepseek/deepseek-reasoner": { - "input_cost_per_token": 5.5e-07, - "input_cost_per_token_cache_hit": 1.4e-07, + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, "litellm_provider": "deepseek", - "max_input_tokens": 65536, + "max_input_tokens": 128000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 2.19e-06, + "output_cost_per_token": 4.2e-07, "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -17481,7 +17482,7 @@ "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17518,7 +17519,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17554,7 +17555,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17863,7 +17864,7 @@ "input_cost_per_token_flex": 6.25e-07, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -17898,9 +17899,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_endpoints": [ @@ -17962,7 +17963,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -17994,7 +17995,7 @@ "input_cost_per_token": 1.25e-06, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18057,7 +18058,7 @@ "input_cost_per_token": 2.5e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18092,7 +18093,7 @@ "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18131,7 +18132,7 @@ "input_cost_per_token_flex": 1.25e-07, "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18169,7 +18170,7 @@ "input_cost_per_token_flex": 2.5e-08, "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -18204,7 +18205,7 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23217,9 +23218,9 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, "supported_modalities": [ @@ -23236,7 +23237,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23255,7 +23256,7 @@ "cache_read_input_token_cost": 1.25e-07, "input_cost_per_token": 1.25e-06, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23274,7 +23275,7 @@ "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 2.5e-07, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -23293,7 +23294,7 @@ "cache_read_input_token_cost": 5e-09, "input_cost_per_token": 5e-08, "litellm_provider": "openrouter", - "max_input_tokens": 272000, + "max_input_tokens": 400000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", @@ -32563,8 +32564,8 @@ "novita/deepseek/deepseek-v3.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.69e-03, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 2.69e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 163840, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32573,15 +32574,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.345e-03, - "input_cost_per_token_cache_hit": 1.345e-03, + "cache_read_input_token_cost": 1.345e-7, + "input_cost_per_token_cache_hit": 1.345e-7, "supports_reasoning": true }, "novita/minimax/minimax-m2.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32590,15 +32591,14 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 3e-04, - "input_cost_per_token_cache_hit": 3e-04, - "supports_reasoning": true + "cache_read_input_token_cost": 3e-8, + "input_cost_per_token_cache_hit": 3e-8 }, "novita/zai-org/glm-4.7": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6e-03, - "output_cost_per_token": 2.2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32607,15 +32607,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.1e-03, - "input_cost_per_token_cache_hit": 1.1e-03, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/xiaomimimo/mimo-v2-flash": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1e-03, - "output_cost_per_token": 3e-03, + "input_cost_per_token": 1e-7, + "output_cost_per_token": 3e-7, "max_input_tokens": 262144, "max_output_tokens": 32000, "max_tokens": 32000, @@ -32624,15 +32624,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2e-04, - "input_cost_per_token_cache_hit": 2e-04, + "cache_read_input_token_cost": 2e-8, + "input_cost_per_token_cache_hit": 2e-8, "supports_reasoning": true }, "novita/zai-org/autoglm-phone-9b-multilingual": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.5e-04, - "output_cost_per_token": 1.38e-03, + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.38e-7, "max_input_tokens": 65536, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32642,8 +32642,8 @@ "novita/moonshotai/kimi-k2-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, @@ -32657,8 +32657,8 @@ "novita/minimax/minimax-m2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 9.6e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32666,15 +32666,15 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "cache_read_input_token_cost": 2.4e-04, - "input_cost_per_token_cache_hit": 2.4e-04, + "cache_read_input_token_cost": 3e-8, + "input_cost_per_token_cache_hit": 3e-8, "supports_reasoning": true }, "novita/paddlepaddle/paddleocr-vl": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-04, - "output_cost_per_token": 1.6e-04, + "input_cost_per_token": 2e-8, + "output_cost_per_token": 2e-8, "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, @@ -32684,8 +32684,8 @@ "novita/deepseek/deepseek-v3.2-exp": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 3.28e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 4.1e-7, "max_input_tokens": 163840, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32699,8 +32699,8 @@ "novita/qwen/qwen3-vl-235b-a22b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.84e-03, - "output_cost_per_token": 3.16e-02, + "input_cost_per_token": 9.8e-7, + "output_cost_per_token": 3.95e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32711,8 +32711,8 @@ "novita/zai-org/glm-4.6v": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3e-03, - "output_cost_per_token": 9e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 9e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32722,15 +32722,15 @@ "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 5.5e-04, - "input_cost_per_token_cache_hit": 5.5e-04, + "cache_read_input_token_cost": 5.5e-8, + "input_cost_per_token_cache_hit": 5.5e-8, "supports_reasoning": true }, "novita/zai-org/glm-4.6": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.4e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 5.5e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32739,15 +32739,31 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, + "novita/kwaipilot/kat-coder-pro": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.2e-06, + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_system_messages": true, + "supports_response_schema": true, + "cache_read_input_token_cost": 6e-8, + "input_cost_per_token_cache_hit": 6e-8 + }, "novita/qwen/qwen3-next-80b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32760,8 +32776,8 @@ "novita/qwen/qwen3-next-80b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32775,8 +32791,8 @@ "novita/deepseek/deepseek-ocr": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 2.4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 3e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -32788,8 +32804,8 @@ "novita/deepseek/deepseek-v3.1-terminus": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32798,15 +32814,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03, + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7, "supports_reasoning": true }, "novita/qwen/qwen3-vl-235b-a22b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 1.2e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.5e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32820,8 +32836,8 @@ "novita/qwen/qwen3-max": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.688e-02, - "output_cost_per_token": 6.76e-02, + "input_cost_per_token": 2.11e-06, + "output_cost_per_token": 8.45e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32834,8 +32850,8 @@ "novita/skywork/r1v4-lite": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-03, - "output_cost_per_token": 6e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 6e-7, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32847,8 +32863,8 @@ "novita/deepseek/deepseek-v3.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32857,15 +32873,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03, + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7, "supports_reasoning": true }, "novita/moonshotai/kimi-k2-0905": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 262144, "max_output_tokens": 262144, "max_tokens": 262144, @@ -32878,8 +32894,8 @@ "novita/qwen/qwen3-coder-480b-a35b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 1.04e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 1.3e-06, "max_input_tokens": 262144, "max_output_tokens": 65536, "max_tokens": 65536, @@ -32892,8 +32908,8 @@ "novita/qwen/qwen3-coder-30b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7e-04, - "output_cost_per_token": 2.7e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.7e-7, "max_input_tokens": 160000, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32906,8 +32922,8 @@ "novita/openai/gpt-oss-120b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 2e-03, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 2.5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32922,8 +32938,8 @@ "novita/moonshotai/kimi-k2-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.56e-03, - "output_cost_per_token": 1.84e-02, + "input_cost_per_token": 5.7e-7, + "output_cost_per_token": 2.3e-06, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, @@ -32936,8 +32952,8 @@ "novita/deepseek/deepseek-v3-0324": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.16e-03, - "output_cost_per_token": 8.96e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 1.12e-06, "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, @@ -32946,14 +32962,14 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 1.08e-03, - "input_cost_per_token_cache_hit": 1.08e-03 + "cache_read_input_token_cost": 1.35e-7, + "input_cost_per_token_cache_hit": 1.35e-7 }, "novita/zai-org/glm-4.5": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 131072, "max_output_tokens": 98304, "max_tokens": 98304, @@ -32961,15 +32977,15 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true, "supports_system_messages": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/qwen/qwen3-235b-a22b-thinking-2507": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 2.4e-02, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 3e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -32982,8 +32998,8 @@ "novita/meta-llama/llama-3.1-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 2e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 16384, "max_output_tokens": 16384, "max_tokens": 16384, @@ -32992,8 +33008,8 @@ "novita/google/gemma-3-12b-it": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 8e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 1e-7, "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33005,8 +33021,8 @@ "novita/zai-org/glm-4.5v": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-03, - "output_cost_per_token": 1.44e-02, + "input_cost_per_token": 6e-7, + "output_cost_per_token": 1.8e-06, "max_input_tokens": 65536, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33016,15 +33032,15 @@ "supports_vision": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 8.8e-04, - "input_cost_per_token_cache_hit": 8.8e-04, + "cache_read_input_token_cost": 1.1e-7, + "input_cost_per_token_cache_hit": 1.1e-7, "supports_reasoning": true }, "novita/openai/gpt-oss-20b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 1.2e-03, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 1.5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33037,8 +33053,8 @@ "novita/qwen/qwen3-235b-a22b-instruct-2507": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 4.64e-03, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 5.8e-7, "max_input_tokens": 131072, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33051,8 +33067,8 @@ "novita/deepseek/deepseek-r1-distill-qwen-14b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.2e-03, - "output_cost_per_token": 1.2e-03, + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 1.5e-7, "max_input_tokens": 32768, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33064,8 +33080,8 @@ "novita/meta-llama/llama-3.3-70b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.08e-03, - "output_cost_per_token": 3.2e-03, + "input_cost_per_token": 1.35e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 131072, "max_output_tokens": 120000, "max_tokens": 120000, @@ -33077,8 +33093,8 @@ "novita/qwen/qwen-2.5-72b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.04e-03, - "output_cost_per_token": 3.2e-03, + "input_cost_per_token": 3.8e-7, + "output_cost_per_token": 4e-7, "max_input_tokens": 32000, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33091,8 +33107,8 @@ "novita/mistralai/mistral-nemo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 1.36e-03, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 1.7e-7, "max_input_tokens": 60288, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33103,8 +33119,8 @@ "novita/minimaxai/minimax-m1-80k": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.4e-03, - "output_cost_per_token": 1.76e-02, + "input_cost_per_token": 5.5e-7, + "output_cost_per_token": 2.2e-06, "max_input_tokens": 1000000, "max_output_tokens": 40000, "max_tokens": 40000, @@ -33117,8 +33133,8 @@ "novita/deepseek/deepseek-r1-0528": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 163840, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33127,15 +33143,15 @@ "supports_tool_choice": true, "supports_system_messages": true, "supports_response_schema": true, - "cache_read_input_token_cost": 2.8e-03, - "input_cost_per_token_cache_hit": 2.8e-03, + "cache_read_input_token_cost": 3.5e-7, + "input_cost_per_token_cache_hit": 3.5e-7, "supports_reasoning": true }, "novita/deepseek/deepseek-r1-distill-qwen-32b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-03, - "output_cost_per_token": 2.4e-03, + "input_cost_per_token": 3e-7, + "output_cost_per_token": 3e-7, "max_input_tokens": 64000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33147,8 +33163,8 @@ "novita/meta-llama/llama-3-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-04, - "output_cost_per_token": 3.2e-04, + "input_cost_per_token": 4e-8, + "output_cost_per_token": 4e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33157,8 +33173,8 @@ "novita/microsoft/wizardlm-2-8x22b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.96e-03, - "output_cost_per_token": 4.96e-03, + "input_cost_per_token": 6.2e-7, + "output_cost_per_token": 6.2e-7, "max_input_tokens": 65535, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33167,8 +33183,8 @@ "novita/deepseek/deepseek-r1-0528-qwen3-8b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 4.8e-04, - "output_cost_per_token": 7.2e-04, + "input_cost_per_token": 6e-8, + "output_cost_per_token": 9e-8, "max_input_tokens": 128000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33178,8 +33194,8 @@ "novita/deepseek/deepseek-r1-distill-llama-70b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 8e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33191,8 +33207,8 @@ "novita/meta-llama/llama-3-70b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.1e-03, - "output_cost_per_token": 7.4e-03, + "input_cost_per_token": 5.1e-7, + "output_cost_per_token": 7.4e-7, "max_input_tokens": 8192, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33203,8 +33219,8 @@ "novita/qwen/qwen3-235b-a22b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33214,8 +33230,8 @@ "novita/meta-llama/llama-4-maverick-17b-128e-instruct-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 7.2e-03, + "input_cost_per_token": 2.7e-7, + "output_cost_per_token": 8.5e-7, "max_input_tokens": 1048576, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33225,8 +33241,8 @@ "novita/meta-llama/llama-4-scout-17b-16e-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 8e-04, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 1.8e-7, + "output_cost_per_token": 5.9e-7, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, @@ -33236,8 +33252,8 @@ "novita/nousresearch/hermes-2-pro-llama-3-8b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.4e-03, - "output_cost_per_token": 1.4e-03, + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 1.4e-7, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33248,8 +33264,8 @@ "novita/qwen/qwen2.5-vl-72b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-03, - "output_cost_per_token": 6.4e-03, + "input_cost_per_token": 8e-7, + "output_cost_per_token": 8e-7, "max_input_tokens": 32768, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33259,8 +33275,8 @@ "novita/sao10k/l3-70b-euryale-v2.1": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.48e-02, - "output_cost_per_token": 1.48e-02, + "input_cost_per_token": 1.48e-06, + "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33272,8 +33288,8 @@ "novita/baidu/ernie-4.5-21B-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 2.24e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.8e-7, "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, @@ -33283,8 +33299,8 @@ "novita/sao10k/l3-8b-lunaris": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33295,33 +33311,18 @@ "novita/baichuan/baichuan-m2-32b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 7e-8, "max_input_tokens": 131072, "max_output_tokens": 131072, "max_tokens": 131072, - "supports_tool_choice": true, - "supports_system_messages": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "novita/thudm/glm-4.1v-9b-thinking": { - "litellm_provider": "novita", - "mode": "chat", - "input_cost_per_token": 2.8e-04, - "output_cost_per_token": 1.104e-03, - "max_input_tokens": 65536, - "max_output_tokens": 8000, - "max_tokens": 8000, - "supports_vision": true, - "supports_system_messages": true, - "supports_reasoning": true + "supports_system_messages": true }, "novita/baidu/ernie-4.5-vl-424b-a47b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.36e-03, - "output_cost_per_token": 1e-02, + "input_cost_per_token": 4.2e-7, + "output_cost_per_token": 1.25e-06, "max_input_tokens": 123000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33332,8 +33333,8 @@ "novita/baidu/ernie-4.5-300b-a47b-paddle": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.24e-03, - "output_cost_per_token": 8.8e-03, + "input_cost_per_token": 2.8e-7, + "output_cost_per_token": 1.1e-06, "max_input_tokens": 123000, "max_output_tokens": 12000, "max_tokens": 12000, @@ -33344,8 +33345,8 @@ "novita/deepseek/deepseek-prover-v2-671b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 160000, "max_output_tokens": 160000, "max_tokens": 160000, @@ -33354,8 +33355,8 @@ "novita/qwen/qwen3-32b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 8e-04, - "output_cost_per_token": 3.6e-03, + "input_cost_per_token": 1e-7, + "output_cost_per_token": 4.5e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33365,8 +33366,8 @@ "novita/qwen/qwen3-30b-a3b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 3.6e-03, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 4.5e-7, "max_input_tokens": 40960, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33376,8 +33377,8 @@ "novita/google/gemma-3-27b-it": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 9.52e-04, - "output_cost_per_token": 1.6e-03, + "input_cost_per_token": 1.19e-7, + "output_cost_per_token": 2e-7, "max_input_tokens": 98304, "max_output_tokens": 16384, "max_tokens": 16384, @@ -33387,8 +33388,8 @@ "novita/deepseek/deepseek-v3-turbo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.2e-03, - "output_cost_per_token": 1.04e-02, + "input_cost_per_token": 4e-7, + "output_cost_per_token": 1.3e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33400,8 +33401,8 @@ "novita/deepseek/deepseek-r1-turbo": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-03, - "output_cost_per_token": 2e-02, + "input_cost_per_token": 7e-7, + "output_cost_per_token": 2.5e-06, "max_input_tokens": 64000, "max_output_tokens": 16000, "max_tokens": 16000, @@ -33414,8 +33415,8 @@ "novita/Sao10K/L3-8B-Stheno-v3.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5e-04, - "output_cost_per_token": 5e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 8192, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33427,8 +33428,8 @@ "novita/gryphe/mythomax-l2-13b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 7.2e-04, - "output_cost_per_token": 7.2e-04, + "input_cost_per_token": 9e-8, + "output_cost_per_token": 9e-8, "max_input_tokens": 4096, "max_output_tokens": 3200, "max_tokens": 3200, @@ -33437,8 +33438,8 @@ "novita/baidu/ernie-4.5-vl-28b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 3.9e-03, - "output_cost_per_token": 3.9e-03, + "input_cost_per_token": 3.9e-7, + "output_cost_per_token": 3.9e-7, "max_input_tokens": 131072, "max_output_tokens": 65536, "max_tokens": 65536, @@ -33453,8 +33454,8 @@ "novita/qwen/qwen3-vl-8b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 6.4e-04, - "output_cost_per_token": 4e-03, + "input_cost_per_token": 8e-8, + "output_cost_per_token": 5e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33468,8 +33469,8 @@ "novita/zai-org/glm-4.5-air": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.04e-03, - "output_cost_per_token": 6.8e-03, + "input_cost_per_token": 1.3e-7, + "output_cost_per_token": 8.5e-7, "max_input_tokens": 131072, "max_output_tokens": 98304, "max_tokens": 98304, @@ -33482,8 +33483,8 @@ "novita/qwen/qwen3-vl-30b-a3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 5.6e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 7e-7, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33497,8 +33498,8 @@ "novita/qwen/qwen3-vl-30b-a3b-thinking": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.6e-03, - "output_cost_per_token": 8e-03, + "input_cost_per_token": 2e-7, + "output_cost_per_token": 1e-06, "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, @@ -33509,11 +33510,45 @@ "supports_system_messages": true, "supports_response_schema": true }, + "novita/qwen/qwen3-omni-30b-a3b-thinking": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 9.7e-7, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_audio_input": true + }, + "novita/qwen/qwen3-omni-30b-a3b-instruct": { + "litellm_provider": "novita", + "mode": "chat", + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 9.7e-7, + "max_input_tokens": 65536, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_system_messages": true, + "supports_response_schema": true, + "supports_audio_input": true, + "supports_audio_output": true + }, "novita/qwen/qwen-mt-plus": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2e-03, - "output_cost_per_token": 6e-03, + "input_cost_per_token": 2.5e-7, + "output_cost_per_token": 7.5e-7, "max_input_tokens": 16384, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33522,8 +33557,8 @@ "novita/baidu/ernie-4.5-vl-28b-a3b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.12e-03, - "output_cost_per_token": 4.48e-03, + "input_cost_per_token": 1.4e-7, + "output_cost_per_token": 5.6e-7, "max_input_tokens": 30000, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33537,8 +33572,8 @@ "novita/baidu/ernie-4.5-21B-a3b": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 2.24e-03, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 2.8e-7, "max_input_tokens": 120000, "max_output_tokens": 8000, "max_tokens": 8000, @@ -33550,8 +33585,8 @@ "novita/qwen/qwen3-8b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.8e-04, - "output_cost_per_token": 1.104e-03, + "input_cost_per_token": 3.5e-8, + "output_cost_per_token": 1.38e-7, "max_input_tokens": 128000, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33561,8 +33596,8 @@ "novita/qwen/qwen3-4b-fp8": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 2.4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 3e-8, "max_input_tokens": 128000, "max_output_tokens": 20000, "max_tokens": 20000, @@ -33572,8 +33607,8 @@ "novita/qwen/qwen2.5-7b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 5.6e-04, - "output_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, + "output_cost_per_token": 7e-8, "max_input_tokens": 32000, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33586,8 +33621,8 @@ "novita/meta-llama/llama-3.2-3b-instruct": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 2.4e-04, - "output_cost_per_token": 4e-04, + "input_cost_per_token": 3e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 32768, "max_output_tokens": 32000, "max_tokens": 32000, @@ -33599,8 +33634,8 @@ "novita/sao10k/l31-70b-euryale-v2.2": { "litellm_provider": "novita", "mode": "chat", - "input_cost_per_token": 1.48e-02, - "output_cost_per_token": 1.48e-02, + "input_cost_per_token": 1.48e-06, + "output_cost_per_token": 1.48e-06, "max_input_tokens": 8192, "max_output_tokens": 8192, "max_tokens": 8192, @@ -33612,7 +33647,7 @@ "novita/qwen/qwen3-embedding-0.6b": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, "output_cost_per_token": 0, "max_input_tokens": 32768, "max_output_tokens": 32768, @@ -33621,7 +33656,7 @@ "novita/qwen/qwen3-embedding-8b": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 5.6e-04, + "input_cost_per_token": 7e-8, "output_cost_per_token": 0, "max_input_tokens": 32768, "max_output_tokens": 4096, @@ -33630,8 +33665,8 @@ "novita/baai/bge-m3": { "litellm_provider": "novita", "mode": "embedding", - "input_cost_per_token": 1e-04, - "output_cost_per_token": 1e-04, + "input_cost_per_token": 1e-8, + "output_cost_per_token": 1e-8, "max_input_tokens": 8192, "max_output_tokens": 96000, "max_tokens": 96000 @@ -33639,8 +33674,8 @@ "novita/qwen/qwen3-reranker-8b": { "litellm_provider": "novita", "mode": "rerank", - "input_cost_per_token": 4e-04, - "output_cost_per_token": 4e-04, + "input_cost_per_token": 5e-8, + "output_cost_per_token": 5e-8, "max_input_tokens": 32768, "max_output_tokens": 4096, "max_tokens": 4096 @@ -33648,8 +33683,8 @@ "novita/baai/bge-reranker-v2-m3": { "litellm_provider": "novita", "mode": "rerank", - "input_cost_per_token": 1e-04, - "output_cost_per_token": 1e-04, + "input_cost_per_token": 1e-8, + "output_cost_per_token": 1e-8, "max_input_tokens": 8000, "max_output_tokens": 8000, "max_tokens": 8000