From 495b3dc026c3a656c3aed3987f23377cda45dd8c Mon Sep 17 00:00:00 2001 From: Utkash Dubey Date: Mon, 10 Mar 2025 14:52:30 -0700 Subject: [PATCH] feedback --- model_prices_and_context_window.json | 2 +- ..._model_prices_and_context_window_schema.py | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 tests/test_model_prices_and_context_window_schema.py diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b2a08544f9..9000a24d06 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6,7 +6,7 @@ "input_cost_per_token": 0.0000, "output_cost_per_token": 0.000, "litellm_provider": "one of https://docs.litellm.ai/docs/providers", - "mode": "one of chat, embedding, completion, image_generation, audio_transcription, audio_speech", + "mode": "one of: chat, embedding, completion, image_generation, audio_transcription, audio_speech, image_generation, moderation, moderations, rerank", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, diff --git a/tests/test_model_prices_and_context_window_schema.py b/tests/test_model_prices_and_context_window_schema.py new file mode 100644 index 0000000000..8efc6c664a --- /dev/null +++ b/tests/test_model_prices_and_context_window_schema.py @@ -0,0 +1,113 @@ +import os +import litellm +from jsonschema import validate + +def test_model_prices_and_context_window_json_is_valid(): + ''' + Validates the `model_prices_and_context_window.json` file. + + If this test fails after you update the json, you need to update the schema or correct the change you made. + ''' + + INTENDED_SCHEMA = { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "cache_creation_input_audio_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "deprecation_date": {"type": "string"}, + "input_cost_per_audio_per_second": {"type": "number"}, + "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_character": {"type": "number"}, + "input_cost_per_character_above_128k_tokens": {"type": "number"}, + "input_cost_per_image": {"type": "number"}, + "input_cost_per_image_above_128k_tokens": {"type": "number"}, + "input_cost_per_pixel": {"type": "number"}, + "input_cost_per_query": {"type": "number"}, + "input_cost_per_request": {"type": "number"}, + "input_cost_per_second": {"type": "number"}, + "input_cost_per_token": {"type": "number"}, + "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_token_batch_requests": {"type": "number"}, + "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_token_cache_hit": {"type": "number"}, + "input_cost_per_video_per_second": {"type": "number"}, + "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, + "input_dbu_cost_per_token": {"type": "number"}, + "litellm_provider": {"type": "string"}, + "max_audio_length_hours": {"type": "number"}, + "max_audio_per_prompt": {"type": "number"}, + "max_document_chunks_per_query": {"type": "number"}, + "max_images_per_prompt": {"type": "number"}, + "max_input_tokens": {"type": "number"}, + "max_output_tokens": {"type": "number"}, + "max_pdf_size_mb": {"type": "number"}, + "max_query_tokens": {"type": "number"}, + "max_tokens": {"type": "number"}, + "max_tokens_per_document_chunk": {"type": "number"}, + "max_video_length": {"type": "number"}, + "max_videos_per_prompt": {"type": "number"}, + "metadata": {"type": "object"}, + "mode": { + "type": "string", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "embedding", + "image_generation", + "moderation", + "moderations", + "rerank" + ], + }, + "output_cost_per_audio_token": {"type": "number"}, + "output_cost_per_character": {"type": "number"}, + "output_cost_per_character_above_128k_tokens": {"type": "number"}, + "output_cost_per_image": {"type": "number"}, + "output_cost_per_pixel": {"type": "number"}, + "output_cost_per_second": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_token_above_128k_tokens": {"type": "number"}, + "output_cost_per_token_batches": {"type": "number"}, + "output_db_cost_per_token": {"type": "number"}, + "output_dbu_cost_per_token": {"type": "number"}, + "output_vector_size": {"type": "number"}, + "rpd": {"type": "number"}, + "rpm": {"type": "number"}, + "source": {"type": "string"}, + "supports_assistant_prefill": {"type": "boolean"}, + "supports_audio_input": {"type": "boolean"}, + "supports_audio_output": {"type": "boolean"}, + "supports_embedding_image_input": {"type": "boolean"}, + "supports_function_calling": {"type": "boolean"}, + "supports_image_input": {"type": "boolean"}, + "supports_parallel_function_calling": {"type": "boolean"}, + "supports_pdf_input": {"type": "boolean"}, + "supports_prompt_caching": {"type": "boolean"}, + "supports_response_schema": {"type": "boolean"}, + "supports_system_messages": {"type": "boolean"}, + "supports_tool_choice": {"type": "boolean"}, + "supports_video_input": {"type": "boolean"}, + "supports_vision": {"type": "boolean"}, + "tool_use_system_prompt_tokens": {"type": "number"}, + "tpm": {"type": "number"}, + }, + "additionalProperties": False, + }, + } + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + actual_json = litellm.get_model_cost_map(url='') + assert isinstance(actual_json, dict) + temporarily_removed = actual_json.pop('sample_spec', None) # remove the sample, whose schema is inconsistent with the real data + + validate(actual_json, INTENDED_SCHEMA) + + if temporarily_removed is not None: + # put back the sample spec that we removed + actual_json.update({'sample_spec': temporarily_removed})