diff --git a/litellm/llms/openai/responses/count_tokens/handler.py b/litellm/llms/openai/responses/count_tokens/handler.py index fba74f3768..e417dbd2a6 100644 --- a/litellm/llms/openai/responses/count_tokens/handler.py +++ b/litellm/llms/openai/responses/count_tokens/handler.py @@ -4,6 +4,7 @@ OpenAI Responses API token counting handler. Uses httpx for HTTP requests to OpenAI's /v1/responses/input_tokens endpoint. """ +import json from typing import Any, Dict, List, Optional, Union import httpx @@ -96,7 +97,7 @@ class OpenAICountTokensHandler(OpenAICountTokensConfig): status_code=e.response.status_code, message=e.response.text, ) - except Exception as e: + except (httpx.RequestError, json.JSONDecodeError) as e: verbose_logger.error(f"Error in CountTokens handler: {str(e)}") raise OpenAIError( status_code=500, diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 5a31973461..a89e8dd743 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -98,7 +98,7 @@ class OpenAICountTokensConfig: from system/developer messages. """ input_items: List[Dict[str, Any]] = [] - instructions: Optional[str] = None + instructions_parts: List[str] = [] for msg in messages: role = msg.get("role", "") @@ -107,7 +107,7 @@ class OpenAICountTokensConfig: if role in ("system", "developer"): # Extract system/developer messages as instructions if isinstance(content, str): - instructions = content + instructions_parts.append(content) elif isinstance(content, list): # Handle content blocks - extract text text_parts = [] @@ -116,11 +116,23 @@ class OpenAICountTokensConfig: text_parts.append(block.get("text", "")) elif isinstance(block, str): text_parts.append(block) - instructions = "\n".join(text_parts) + instructions_parts.append("\n".join(text_parts)) elif role == "user": input_items.append({"role": "user", "content": content}) elif role == "assistant": - input_items.append({"role": "assistant", "content": content}) + # Map tool_calls to Responses API function_call items + tool_calls = msg.get("tool_calls") + if tool_calls: + for tc in tool_calls: + func = tc.get("function", {}) + input_items.append({ + "type": "function_call", + "call_id": tc.get("id", ""), + "name": func.get("name", ""), + "arguments": func.get("arguments", ""), + }) + else: + input_items.append({"role": "assistant", "content": content}) elif role == "tool": input_items.append({ "type": "function_call_output", @@ -128,4 +140,5 @@ class OpenAICountTokensConfig: "output": content if isinstance(content, str) else str(content), }) + instructions = "\n".join(instructions_parts) if instructions_parts else None return input_items, instructions diff --git a/litellm/main.py b/litellm/main.py index 9f462a4b3f..e70055a90a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7609,10 +7609,12 @@ async def acount_tokens( tools=tools, system=system, ) - if result is not None: + if result is not None and not result.error: return result - except Exception: - pass + except Exception as e: + verbose_logger.debug( + f"Provider token counting failed for model={model}, falling back to local: {e}" + ) # Fallback to local tiktoken-based token counting local_count = litellm.token_counter( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 46ad2d0d60..abfc25acbc 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -517,7 +517,7 @@ async def count_response_input_tokens( ) ) raise HTTPException( - status_code=500, detail={"error": f"Internal server error: {str(e)}"} + status_code=500, detail={"error": "Internal server error"} ) diff --git a/tests/test_litellm/test_count_tokens_public_api.py b/tests/test_litellm/test_count_tokens_public_api.py index 730ddbc23f..59a1092455 100644 --- a/tests/test_litellm/test_count_tokens_public_api.py +++ b/tests/test_litellm/test_count_tokens_public_api.py @@ -133,9 +133,10 @@ def test_acount_tokens_api_error_falls_back(): ) ) - # Should return error response, not raise - assert result.error is True - assert result.status_code == 401 + # Should fall back to local tokenizer when provider API errors + assert result.error is False + assert result.tokenizer_type == "local_tokenizer" + assert result.total_tokens > 0 def test_acount_tokens_no_api_key_falls_back():