feat: Replace jsonpath-ng with custom minimal parser for additional_drop_params

2026-08-02 00:22:06 +00:00 · 2025-12-09 17:30:30 +05:30
parent 761798e5c7
commit 9bb0e7dd75
5 changed files with 279 additions and 57 deletions
@@ -117,6 +117,56 @@ response = litellm.completion(

 **additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model.

+### Nested Field Removal
+
+Drop nested fields within complex objects using JSONPath-like notation:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+    messages=[{"role": "user", "content": "Hello"}],
+    tools=[{
+        "name": "search",
+        "description": "Search files",
+        "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
+        "input_examples": [{"query": "test"}]  # Will be removed
+    }],
+    additional_drop_params=["tools[*].input_examples"]  # Remove from all tools
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+model_list:
+  - model_name: my-bedrock-model
+    litellm_params:
+      model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
+      additional_drop_params: ["tools[*].input_examples"]  # Remove from all tools
+```
+
+</TabItem>
+</Tabs>
+
+**Supported syntax:**
+- `field` - Top-level field
+- `parent.child` - Nested object field
+- `array[*]` - All array elements
+- `array[0]` - Specific array index
+- `tools[*].input_examples` - Field in all array elements
+- `tools[0].metadata.field` - Specific index + nested field
+
+**Example use cases:**
+- Remove `input_examples` from tool definitions (Claude Code + AWS Bedrock)
+- Drop provider-specific fields from nested structures
+- Clean up nested parameters before sending to LLM
+
 ## Specify allowed openai params in a request

 Tell litellm to allow specific openai params in a request. Use this if you get a `litellm.UnsupportedParamsError` and want to allow a param. LiteLLM will pass the param as is to the model.
@@ -2,9 +2,9 @@
 Path-based navigation utilities for nested dictionaries.

 This module provides utilities for reading and deleting values in nested
-dictionaries using dot notation and JSONPath array syntax.
+dictionaries using dot notation and JSONPath-like array syntax.

-Uses jsonpath-ng library for standard JSONPath parsing and navigation.
+Custom implementation with zero external dependencies.

 Supported syntax:
 - "field" - top-level field
@@ -77,6 +77,100 @@ def get_nested_value(
    return current if isinstance(current, type(default)) else default


+def _parse_path_segments(path: str) -> list:
+    """
+    Parse a JSONPath-like string into segments using regex.
+
+    Handles:
+    - Dot notation: "a.b.c" → ["a", "b", "c"]
+    - Array wildcards: "a[*].b" → ["a", "[*]", "b"]
+    - Array indices: "a[0].b" → ["a", "[0]", "b"]
+
+    Args:
+        path: JSONPath-like path string
+
+    Returns:
+        List of path segments
+
+    Example:
+        >>> _parse_path_segments("tools[*].arr[0].field")
+        ["tools", "[*]", "arr", "[0]", "field"]
+    """
+    import re
+
+    # Match field names OR bracket expressions
+    # Pattern: field_name (anything except . or [) | [anything_in_brackets]
+    pattern = r'[^\.\[]+|\[[^\]]*\]'
+    segments = re.findall(pattern, path)
+    return segments
+
+
+def _delete_nested_value_custom(
+    data: Dict[str, Any],
+    segments: list,
+    segment_index: int = 0,
+) -> None:
+    """
+    Recursively delete a field from nested data using parsed segments.
+
+    Modifies data in-place (caller must deep copy first).
+
+    Args:
+        data: Dictionary or list to modify
+        segments: Parsed path segments
+        segment_index: Current position in segments list
+    """
+    if segment_index >= len(segments):
+        return
+
+    segment = segments[segment_index]
+    is_last = segment_index == len(segments) - 1
+
+    # Handle array wildcard: [*]
+    if segment == "[*]":
+        if isinstance(data, list):
+            for item in data:
+                if is_last:
+                    # Can't delete array elements themselves, skip
+                    pass
+                else:
+                    _delete_nested_value_custom(item, segments, segment_index + 1)
+        return
+
+    # Handle array index: [0], [1], [2], etc.
+    if segment.startswith("[") and segment.endswith("]"):
+        try:
+            index = int(segment[1:-1])
+            if isinstance(data, list) and 0 <= index < len(data):
+                if is_last:
+                    # Can't delete array elements themselves, skip
+                    pass
+                else:
+                    _delete_nested_value_custom(data[index], segments, segment_index + 1)
+        except (ValueError, IndexError):
+            # Invalid index, skip
+            pass
+        return
+
+    # Handle regular field navigation
+    if isinstance(data, dict):
+        if is_last:
+            # Delete the field
+            data.pop(segment, None)
+        else:
+            # Navigate deeper
+            if segment in data:
+                next_segment = segments[segment_index + 1] if segment_index + 1 < len(segments) else None
+
+                # If next segment is array notation, current field should be list
+                if next_segment and (next_segment.startswith("[")):
+                    if isinstance(data[segment], list):
+                        _delete_nested_value_custom(data[segment], segments, segment_index + 1)
+                # Otherwise navigate into dict
+                elif isinstance(data[segment], dict):
+                    _delete_nested_value_custom(data[segment], segments, segment_index + 1)
+
+
 def delete_nested_value(
    data: Dict[str, Any],
    path: str,
@@ -86,13 +180,13 @@ def delete_nested_value(
    """
    Delete a field from nested data using JSONPath notation.

-    Uses jsonpath-ng library for standard JSONPath parsing.
+    Custom implementation - no external dependencies.

    Supports:
    - "field" - top-level field
    - "parent.child" - nested field
-    - "array[*]" - all array elements
-    - "array[0]" - specific array element
+    - "array[*]" - all array elements (wildcard)
+    - "array[0]" - specific array element (index)
    - "array[*].field" - field in all array elements

    Args:
@@ -111,31 +205,18 @@ def delete_nested_value(
    """
    import copy

-    from jsonpath_ng import parse
-
    result = copy.deepcopy(data)

-    # Add $ prefix required by jsonpath-ng
-    if not path.startswith("$"):
-        path = f"$.{path}"
-
    try:
-        expr = parse(path)
-        matches = expr.find(result)
+        # Parse path into segments
+        segments = _parse_path_segments(path)

-        # Process matches in reverse to handle array deletions correctly
-        for match in reversed(matches):
-            parent = match.context.value if match.context else result
+        if not segments:
+            return result
+
+        # Delete using custom recursive implementation
+        _delete_nested_value_custom(result, segments, 0)

-            if isinstance(parent, list):
-                if hasattr(match.path, "index"):
-                    idx = match.path.index
-                    if 0 <= idx < len(parent):
-                        parent.pop(idx)
-            elif isinstance(parent, dict):
-                if hasattr(match.path, "fields") and match.path.fields:
-                    field = match.path.fields[0]
-                    parent.pop(field, None)
    except Exception:
        # Invalid path or parsing error - silently skip
        pass
@@ -2195,7 +2195,7 @@ files = [
    {file = "grpcio-1.70.0-cp39-cp39-win_amd64.whl", hash = "sha256:a31d7e3b529c94e930a117b2175b2efd179d96eb3c7a21ccb0289a8ab05b645c"},
    {file = "grpcio-1.70.0.tar.gz", hash = "sha256:8d1584a68d5922330025881e63a6c1b54cc8117291d382e4fa69339b6d914c56"},
 ]
-markers = {main = "python_version < \"3.14\" and extra == \"extra-proxy\"", dev = "python_version < \"3.14\"", proxy-dev = "python_version < \"3.14\""}
+markers = {main = "(python_version < \"3.9\" or platform_python_implementation == \"PyPy\") and extra == \"extra-proxy\" and python_version < \"3.10\"", dev = "python_version < \"3.9\" or python_version < \"3.10\" and platform_python_implementation == \"PyPy\"", proxy-dev = "python_version < \"3.9\" or python_version < \"3.10\" and platform_python_implementation == \"PyPy\""}

 [package.extras]
 protobuf = ["grpcio-tools (>=1.70.0)"]
@@ -2270,7 +2270,7 @@ files = [
    {file = "grpcio-1.76.0-cp39-cp39-win_amd64.whl", hash = "sha256:acab0277c40eff7143c2323190ea57b9ee5fd353d8190ee9652369fae735668a"},
    {file = "grpcio-1.76.0.tar.gz", hash = "sha256:7be78388d6da1a25c0d5ec506523db58b18be22d9c37d8d3a32c08be4987bd73"},
 ]
-markers = {main = "python_version >= \"3.14\" and extra == \"extra-proxy\"", dev = "python_version >= \"3.14\"", proxy-dev = "python_version >= \"3.14\""}
+markers = {main = "(python_version >= \"3.10\" or platform_python_implementation != \"PyPy\") and extra == \"extra-proxy\" and python_version >= \"3.9\"", dev = "platform_python_implementation != \"PyPy\" and python_version >= \"3.9\" or python_version >= \"3.10\"", proxy-dev = "platform_python_implementation != \"PyPy\" and python_version >= \"3.9\" or python_version >= \"3.10\""}

 [package.dependencies]
 typing-extensions = ">=4.12,<5.0"
@@ -2805,22 +2805,6 @@ files = [
    {file = "joblib-1.5.2.tar.gz", hash = "sha256:3faa5c39054b2f03ca547da9b2f52fde67c06240c31853f306aea97f13647b55"},
 ]

-[[package]]
-name = "jsonpath-ng"
-version = "1.7.0"
-description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming."
-optional = false
-python-versions = "*"
-groups = ["main"]
-files = [
-    {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"},
-    {file = "jsonpath_ng-1.7.0-py2-none-any.whl", hash = "sha256:898c93fc173f0c336784a3fa63d7434297544b7198124a68f9a3ef9597b0ae6e"},
-    {file = "jsonpath_ng-1.7.0-py3-none-any.whl", hash = "sha256:f3d7f9e848cba1b6da28c55b1c26ff915dc9e0b1ba7e752a53d6da8d5cbd00b6"},
-]
-
-[package.dependencies]
-ply = "*"
-
 [[package]]
 name = "jsonschema"
 version = "4.23.0"
@@ -4290,18 +4274,6 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]

-[[package]]
-name = "ply"
-version = "3.11"
-description = "Python Lex & Yacc"
-optional = false
-python-versions = "*"
-groups = ["main"]
-files = [
-    {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"},
-    {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"},
-]
-
 [[package]]
 name = "polars"
 version = "1.35.2"
@@ -7123,4 +7095,4 @@ utils = ["numpydoc"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "a76852a460dcd45b1bf39a87ffe9dad6bda3ff4e5655f5ab4b436344c0c5663b"
+content-hash = "ced4152a4f49f03f1909103e4ee1738d4f7c0f04bc9c7ea5f59a84917a4b19d7"
@@ -68,7 +68,6 @@ semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"
 mlflow = {version = ">3.1.4", optional = true, python = ">=3.10"}
 soundfile = {version = "^0.12.1", optional = true}
 grpcio = ">=1.62.3,<1.68.0"  # Constrain to < 1.68.0 to avoid resource exhausted bug (https://github.com/grpc/grpc/issues/38290). Minimum 1.62.3 required by grpcio-status.
-jsonpath-ng = "^1.7.0"

 [tool.poetry.extras]
 proxy = [
@@ -63,6 +63,50 @@ class TestDeleteNestedValue:
        # Verify original unchanged (deep copy)
        assert "input_examples" in data["tools"][0]

+    def test_specific_array_index_removes_field_from_single_element(self):
+        """Test removing a field from specific array elements using [n] syntax."""
+        # Test data with multiple array elements
+        data = {
+            "tools": [
+                {"name": "t0", "input_examples": ["ex0"], "keep": "val0"},
+                {"name": "t1", "input_examples": ["ex1"], "keep": "val1"},
+                {"name": "t2", "input_examples": ["ex2"], "keep": "val2"},
+                {"name": "t3", "input_examples": ["ex3"], "keep": "val3"},
+                {"name": "t4", "input_examples": ["ex4"], "keep": "val4"},
+                {"name": "t5", "input_examples": ["ex5"], "keep": "val5"},
+            ]
+        }
+
+        # Test [0] - first element
+        result = delete_nested_value(data, "tools[0].input_examples")
+        assert "input_examples" not in result["tools"][0]
+        assert "input_examples" in result["tools"][1]
+        assert "input_examples" in result["tools"][2]
+
+        # Test [1] - second element
+        result = delete_nested_value(data, "tools[1].input_examples")
+        assert "input_examples" in result["tools"][0]
+        assert "input_examples" not in result["tools"][1]
+        assert "input_examples" in result["tools"][2]
+
+        # Test [2] - middle element
+        result = delete_nested_value(data, "tools[2].input_examples")
+        assert "input_examples" in result["tools"][0]
+        assert "input_examples" not in result["tools"][2]
+        assert "input_examples" in result["tools"][3]
+
+        # Test [5] - last element
+        result = delete_nested_value(data, "tools[5].input_examples")
+        assert "input_examples" in result["tools"][0]
+        assert "input_examples" not in result["tools"][5]
+
+        # Verify other fields preserved in all cases
+        assert result["tools"][0]["keep"] == "val0"
+        assert result["tools"][5]["keep"] == "val5"
+
+        # Verify original unchanged (deep copy)
+        assert "input_examples" in data["tools"][0]
+

 class TestComplexNestedPatterns:
    """Test complex nested patterns with multiple wildcards and deep nesting."""
@@ -230,5 +274,81 @@ class TestComplexNestedPatterns:
        assert result["top_level_remove"] == "should_go"
        assert result["top_level_keep"] == "should_stay"

+    def test_mixed_wildcards_and_indices_with_deep_nesting(self):
+        """Test combining [*] wildcards, [n] indices, and deep nesting in complex patterns."""
+        data = {
+            "tools": [
+                {
+                    "name": "t0",
+                    "configs": [
+                        {"id": "c0", "remove_me": "val1", "keep": "yes1"},
+                        {"id": "c1", "remove_me": "val2", "keep": "yes2"},
+                    ],
+                    "metadata": {"drop_this": "meta1", "preserve": "preserve1"},
+                },
+                {
+                    "name": "t1",
+                    "configs": [
+                        {"id": "c0", "remove_me": "val3", "keep": "yes3"},
+                        {"id": "c1", "remove_me": "val4", "keep": "yes4"},
+                    ],
+                    "metadata": {"drop_this": "meta2", "preserve": "preserve2"},
+                },
+                {
+                    "name": "t2",
+                    "configs": [
+                        {"id": "c0", "remove_me": "val5", "keep": "yes5"},
+                    ],
+                    "metadata": {"drop_this": "meta3", "preserve": "preserve3"},
+                },
+            ]
+        }
+
+        # Simulate processing multiple complex paths
+        paths = [
+            "tools[*].configs[1].remove_me",  # Wildcard + specific index [1] + nested
+            "tools[1].metadata.drop_this",  # Specific index + nested
+            "tools[*].configs[*].id",  # Double wildcard + nested
+        ]
+
+        result = data
+        for path in paths:
+            result = delete_nested_value(result, path)
+
+        # Verify: tools[*].configs[1].remove_me removed from second config of all tools (that have one)
+        assert "remove_me" in result["tools"][0]["configs"][0]  # First config untouched
+        assert (
+            "remove_me" not in result["tools"][0]["configs"][1]
+        )  # Second config removed
+        assert "remove_me" in result["tools"][1]["configs"][0]  # First config untouched
+        assert (
+            "remove_me" not in result["tools"][1]["configs"][1]
+        )  # Second config removed
+        assert (
+            "remove_me" in result["tools"][2]["configs"][0]
+        )  # Only has [0], unaffected
+
+        # Verify: tools[1].metadata.drop_this removed only from second tool
+        assert "drop_this" in result["tools"][0]["metadata"]
+        assert "drop_this" not in result["tools"][1]["metadata"]
+        assert "drop_this" in result["tools"][2]["metadata"]
+
+        # Verify: tools[*].configs[*].id removed from all configs in all tools
+        assert "id" not in result["tools"][0]["configs"][0]
+        assert "id" not in result["tools"][0]["configs"][1]
+        assert "id" not in result["tools"][1]["configs"][0]
+        assert "id" not in result["tools"][1]["configs"][1]
+        assert "id" not in result["tools"][2]["configs"][0]
+
+        # Verify: other fields preserved
+        assert result["tools"][0]["configs"][0]["keep"] == "yes1"
+        assert result["tools"][1]["configs"][1]["keep"] == "yes4"
+        assert result["tools"][0]["metadata"]["preserve"] == "preserve1"
+        assert result["tools"][1]["metadata"]["preserve"] == "preserve2"
+        assert result["tools"][2]["name"] == "t2"
+
+        # Verify original unchanged
+        assert "remove_me" in data["tools"][0]["configs"][1]
+

 # Phase 1 tests - validates core functionality and complex patterns