docs(input.md): document 'extra_headers' param support (#7268)

* docs(input.md): document 'extra_headers' param support

* fix: #7239 to move Nova topK parameter to `additionalModelRequestFields` (#7240)

Co-authored-by: Ryan Hoium <rhoium>

---------

Co-authored-by: ryanh-ai <3118399+ryanh-ai@users.noreply.github.com>
This commit is contained in:
Krish Dholakia
2024-12-17 07:19:14 -08:00
committed by GitHub
parent f3b13a9af3
commit cd5bdfcb7a
3 changed files with 25 additions and 1 deletions
+4
View File
@@ -191,6 +191,10 @@ def completion(
- `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
- `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
- `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request.
#### Deprecated Params
- `functions`: *array* - A list of functions that the model may use to generate JSON inputs. Each function should have the following properties:
@@ -378,6 +378,12 @@ class AmazonConverseConfig:
for key in additional_request_keys:
inference_params.pop(key, None)
if 'topK' in inference_params:
additional_request_params["inferenceConfig"] = {'topK': inference_params.pop("topK")}
elif 'top_k' in inference_params:
additional_request_params["inferenceConfig"] = {'topK': inference_params.pop("top_k")}
bedrock_tools: List[ToolBlock] = _bedrock_tools_pt(
inference_params.pop("tools", [])
)
@@ -2215,8 +2215,22 @@ def test_bedrock_nova_topk(top_k_param):
"messages": [{"role": "user", "content": "Hello, world!"}],
top_k_param: 10,
}
litellm.completion(**data)
original_transform = litellm.AmazonConverseConfig()._transform_request
captured_data = None
def mock_transform(*args, **kwargs):
nonlocal captured_data
result = original_transform(*args, **kwargs)
captured_data = result
return result
with patch('litellm.AmazonConverseConfig._transform_request', side_effect=mock_transform):
litellm.completion(**data)
# Assert that additionalRequestParameters exists and contains topK
assert 'additionalModelRequestFields' in captured_data
assert 'inferenceConfig' in captured_data['additionalModelRequestFields']
assert captured_data['additionalModelRequestFields']['inferenceConfig']['topK'] == 10
def test_bedrock_process_empty_text_blocks():
from litellm.litellm_core_utils.prompt_templates.factory import (