mirror of
https://github.com/tiennm99/litellm.git
synced 2026-08-01 20:21:14 +00:00
fix(check_licenses): read PEP 639 license-expression metadata (#28529)
The dependency license checker only read the legacy free-text `info.license` field from PyPI. Packages that adopt PEP 639 publish their license as an SPDX expression in `info.license_expression` and leave the legacy field null, so the checker reported "Unknown license" and failed CI for every newly-bumped PEP 639 dependency. `get_package_license_from_pypi` now resolves the license in order: `license_expression`, then legacy `license`, then the `License :: OSI Approved :: ...` trove classifiers. `is_license_acceptable` splits compound SPDX expressions on the uppercase OR/AND operators (case-sensitive, so the lowercase `-or-later` inside an identifier is not mistaken for an operator) and strips `WITH <exception>` suffixes, requiring every component to be acceptable. Free-text license blobs are detected and fall back to the original whole-string matching. The `black` and `pydantic-settings` entries in liccheck.ini that existed solely to work around this now resolve correctly on their own and have been removed.
This commit is contained in:
yuneng-jiang
@@ -31,6 +31,13 @@ DEFAULT_TRANSITIVE_PIN_PACKAGES = (
|
||||
"wheel",
|
||||
)
|
||||
|
||||
# SPDX license expressions (PEP 639 "License-Expression") join identifiers with
|
||||
# the uppercase operators OR / AND / WITH. The split is case-sensitive: the
|
||||
# lowercase "-or-later" inside an identifier such as "GPL-2.0-or-later" is part
|
||||
# of the identifier, not an operator.
|
||||
_SPDX_OPERATOR_SPLIT = re.compile(r"\s+(?:OR|AND)\s+")
|
||||
_SPDX_WITH_SUFFIX = re.compile(r"\s+WITH\s+.*", re.DOTALL)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackageLicense:
|
||||
@@ -109,21 +116,86 @@ class LicenseChecker:
|
||||
def get_package_license_from_pypi(
|
||||
self, package_name: str, version: str
|
||||
) -> Optional[str]:
|
||||
"""Fetch license information for a package from PyPI."""
|
||||
"""Fetch license information for a package from PyPI.
|
||||
|
||||
Prefers the PEP 639 SPDX expression (``info.license_expression``),
|
||||
falls back to the legacy free-text ``info.license`` field, and as a
|
||||
last resort derives the license from the ``License :: OSI Approved ::
|
||||
...`` trove classifiers.
|
||||
"""
|
||||
try:
|
||||
url = f"https://pypi.org/pypi/{package_name}/{version}/json"
|
||||
response = requests.get(url, timeout=10)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data.get("info", {}).get("license")
|
||||
info = response.json().get("info", {}) or {}
|
||||
return (
|
||||
info.get("license_expression")
|
||||
or info.get("license")
|
||||
or self._license_from_classifiers(info.get("classifiers") or [])
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"Warning: Failed to fetch license for {package_name} {version}: {str(e)}"
|
||||
)
|
||||
return None
|
||||
|
||||
def is_license_acceptable(self, license_str: str) -> Tuple[bool, str]:
|
||||
"""Check if a license is acceptable based on configured lists."""
|
||||
@staticmethod
|
||||
def _license_from_classifiers(classifiers: List[str]) -> Optional[str]:
|
||||
"""Derive a license name from the ``License :: OSI Approved :: ...`` trove classifiers."""
|
||||
prefix = "License :: OSI Approved :: "
|
||||
for classifier in classifiers:
|
||||
if classifier.startswith(prefix):
|
||||
license_name = classifier[len(prefix) :].strip()
|
||||
if license_name:
|
||||
return license_name
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _split_spdx_expression(license_str: str) -> Optional[List[str]]:
|
||||
"""Split an SPDX license expression into its component identifiers.
|
||||
|
||||
Returns ``None`` when the string is not a recognizable SPDX expression
|
||||
(for example a free-text license blob), so callers fall back to
|
||||
whole-string matching.
|
||||
"""
|
||||
if "OR" not in license_str and "AND" not in license_str:
|
||||
return None
|
||||
|
||||
components: List[str] = []
|
||||
normalized = license_str.replace("(", " ").replace(")", " ")
|
||||
for part in _SPDX_OPERATOR_SPLIT.split(normalized):
|
||||
# Drop any "WITH <exception>" suffix: the exception qualifies the
|
||||
# preceding license, it is not itself a license to authorize.
|
||||
identifier = _SPDX_WITH_SUFFIX.sub("", part).strip()
|
||||
if not identifier:
|
||||
continue
|
||||
# SPDX short-form identifiers are single whitespace-free tokens; a
|
||||
# component with internal whitespace means this is free text.
|
||||
if any(char.isspace() for char in identifier):
|
||||
return None
|
||||
components.append(identifier)
|
||||
|
||||
return components if len(components) > 1 else None
|
||||
|
||||
def is_license_acceptable(self, license_str: Optional[str]) -> Tuple[bool, str]:
|
||||
"""Check if a license (or compound SPDX expression) is acceptable."""
|
||||
if not license_str:
|
||||
return False, "Unknown license"
|
||||
|
||||
components = self._split_spdx_expression(license_str)
|
||||
if components is None:
|
||||
return self._is_single_license_acceptable(license_str)
|
||||
|
||||
# Compound SPDX expression: conservatively require every component to
|
||||
# be acceptable on its own (the safe direction for a CI gate).
|
||||
for component in components:
|
||||
is_acceptable, reason = self._is_single_license_acceptable(component)
|
||||
if not is_acceptable:
|
||||
return False, f"{reason} (in SPDX expression '{license_str}')"
|
||||
return True, f"All SPDX components authorized: {', '.join(components)}"
|
||||
|
||||
def _is_single_license_acceptable(self, license_str: str) -> Tuple[bool, str]:
|
||||
"""Check if a single license identifier is acceptable based on configured lists."""
|
||||
if not license_str:
|
||||
return False, "Unknown license"
|
||||
|
||||
|
||||
@@ -90,7 +90,6 @@ jinja2: >=3.1.4 # BSD 3-Clause License
|
||||
litellm-proxy-extras: >=0.1.1 # MIT License
|
||||
litellm-enterprise: >=0.1.1 # LiteLLM Enterprise License
|
||||
a2a-sdk: >=0.3.22 # Apache 2.0 license
|
||||
pydantic-settings: >=2.14.1 # MIT License manually verified (uses PEP 639 License-Expression: MIT, not the legacy License field, so liccheck reports it as unknown)
|
||||
anyio: >=4.5.0 # Unknown license
|
||||
httpx-aiohttp: >=0.1.4 # Unknown license
|
||||
backoff: >=2.2.1 # Unknown license
|
||||
@@ -156,7 +155,6 @@ pytest: >=9.0.3 # MIT license
|
||||
pytest-postgresql: >=7.0.2 # LGPLv3+ license
|
||||
pytest-xdist: >=3.8.0 # MIT License
|
||||
ruff: >=0.15.3 # MIT License
|
||||
black: >=26.3.1 # MIT License manually verified (uses PEP 639 License-Expression: MIT, not the legacy License field, so liccheck reports it as unknown)
|
||||
types-requests: >=2.32.4.20260107 # Apache 2.0 license (typeshed)
|
||||
types-pyyaml: >=6.0.12.20250915 # Apache 2.0 license (typeshed)
|
||||
fakeredis: >=2.34.1 # BSD license
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Tests for the dependency license checker at tests/code_coverage_tests/check_licenses.py.
|
||||
|
||||
Focus: PEP 639 license metadata. Packages that adopt PEP 639 publish their
|
||||
license as an SPDX expression in ``info.license_expression`` and often leave the
|
||||
legacy ``info.license`` field null, so the checker must read the new field (and
|
||||
fall back to trove classifiers) instead of reporting "Unknown license".
|
||||
|
||||
PyPI HTTP responses are mocked — these tests never hit the network.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_CODE_COVERAGE_DIR = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "..", "code_coverage_tests"
|
||||
)
|
||||
sys.path.insert(0, _CODE_COVERAGE_DIR)
|
||||
|
||||
import check_licenses # noqa: E402
|
||||
|
||||
_LICCHECK_INI = Path(_CODE_COVERAGE_DIR) / "liccheck.ini"
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
def _make_checker():
|
||||
return check_licenses.LicenseChecker(config_file=_LICCHECK_INI)
|
||||
|
||||
|
||||
def _patch_pypi(monkeypatch, info):
|
||||
"""Make PyPI return a JSON response with the given ``info`` block."""
|
||||
|
||||
def _fake_get(url, timeout=None):
|
||||
return _FakeResponse({"info": info})
|
||||
|
||||
monkeypatch.setattr(check_licenses.requests, "get", _fake_get)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# get_package_license_from_pypi: license metadata resolution
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_license_prefers_license_expression(monkeypatch):
|
||||
"""(a) PEP 639 packages publish the SPDX expression in license_expression."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": "MIT", "license": None, "classifiers": []},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.get_package_license_from_pypi("black", "26.3.1") == "MIT"
|
||||
|
||||
|
||||
def test_license_expression_wins_when_both_present(monkeypatch):
|
||||
"""license_expression takes precedence over the legacy license field."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": "Apache-2.0", "license": "stale free text"},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.get_package_license_from_pypi("pkg", "1.0.0") == "Apache-2.0"
|
||||
|
||||
|
||||
def test_get_license_falls_back_to_legacy_license(monkeypatch):
|
||||
"""(b) Pre-PEP-639 packages only set the legacy free-text license field."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": None, "license": "MIT License", "classifiers": []},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.get_package_license_from_pypi("pkg", "1.0.0") == "MIT License"
|
||||
|
||||
|
||||
def test_get_license_falls_back_to_classifiers(monkeypatch):
|
||||
"""(c) Some packages express the license only through trove classifiers."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{
|
||||
"license_expression": None,
|
||||
"license": None,
|
||||
"classifiers": [
|
||||
"Programming Language :: Python :: 3",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
],
|
||||
},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert (
|
||||
checker.get_package_license_from_pypi("pkg", "1.0.0")
|
||||
== "Apache Software License"
|
||||
)
|
||||
|
||||
|
||||
def test_get_license_returns_none_when_unset(monkeypatch):
|
||||
"""(d) With no license metadata at all the license stays unknown."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": None, "license": None, "classifiers": []},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None
|
||||
|
||||
|
||||
def test_get_license_returns_none_on_request_failure(monkeypatch):
|
||||
"""Network/HTTP failures are swallowed and reported as unknown."""
|
||||
|
||||
def _boom(url, timeout=None):
|
||||
raise RuntimeError("network down")
|
||||
|
||||
monkeypatch.setattr(check_licenses.requests, "get", _boom)
|
||||
checker = _make_checker()
|
||||
assert checker.get_package_license_from_pypi("pkg", "1.0.0") is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# is_license_acceptable: SPDX identifiers and compound expressions
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_spdx_identifiers_are_authorized():
|
||||
"""Plain SPDX identifiers match the legacy-spelled authorized list as-is."""
|
||||
checker = _make_checker()
|
||||
for identifier in ("MIT", "Apache-2.0", "BSD-3-Clause"):
|
||||
is_ok, reason = checker.is_license_acceptable(identifier)
|
||||
assert is_ok is True, f"{identifier}: {reason}"
|
||||
|
||||
|
||||
def test_spdx_compound_or_expression_is_authorized():
|
||||
checker = _make_checker()
|
||||
is_ok, reason = checker.is_license_acceptable("MIT OR Apache-2.0")
|
||||
assert is_ok is True, reason
|
||||
|
||||
|
||||
def test_spdx_with_exception_in_compound_is_authorized():
|
||||
"""The 'WITH <exception>' suffix is stripped; the base license is checked."""
|
||||
checker = _make_checker()
|
||||
is_ok, reason = checker.is_license_acceptable(
|
||||
"Apache-2.0 WITH LLVM-exception OR MIT"
|
||||
)
|
||||
assert is_ok is True, reason
|
||||
|
||||
|
||||
def test_spdx_gpl3_is_rejected():
|
||||
"""GPL-3.0 spellings must fail — they match no authorized license."""
|
||||
checker = _make_checker()
|
||||
for expr in ("GPL-3.0-only", "GPL-3.0-or-later"):
|
||||
is_ok, reason = checker.is_license_acceptable(expr)
|
||||
assert is_ok is False, f"{expr} unexpectedly accepted: {reason}"
|
||||
|
||||
|
||||
def test_spdx_compound_with_copyleft_component_is_rejected():
|
||||
"""A permissive-OR-copyleft expression is conservatively rejected."""
|
||||
checker = _make_checker()
|
||||
is_ok, _ = checker.is_license_acceptable("MIT OR GPL-3.0-only")
|
||||
assert is_ok is False
|
||||
|
||||
|
||||
def test_or_later_identifier_is_not_split_as_operator():
|
||||
"""The lowercase '-or-later' inside an identifier is not the SPDX OR operator."""
|
||||
assert (
|
||||
check_licenses.LicenseChecker._split_spdx_expression("GPL-2.0-or-later") is None
|
||||
)
|
||||
|
||||
|
||||
def test_free_text_license_is_not_treated_as_spdx():
|
||||
"""Free-text license blobs fall back to whole-string substring matching."""
|
||||
free_text = "MIT License AND additional redistribution permissions"
|
||||
assert check_licenses.LicenseChecker._split_spdx_expression(free_text) is None
|
||||
checker = _make_checker()
|
||||
assert checker.is_license_acceptable(free_text)[0] is True
|
||||
|
||||
|
||||
def test_unknown_license_is_reported():
|
||||
checker = _make_checker()
|
||||
is_ok, reason = checker.is_license_acceptable(None)
|
||||
assert is_ok is False
|
||||
assert reason == "Unknown license"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# check_package: end-to-end resolution + acceptability
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_package_accepts_pep639_package(monkeypatch):
|
||||
"""A PEP 639 package whose license lives only in license_expression passes."""
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": "MIT", "license": None, "classifiers": []},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.check_package("some-pep639-pkg", "1.0.0") is True
|
||||
|
||||
|
||||
def test_check_package_rejects_package_without_license(monkeypatch):
|
||||
_patch_pypi(
|
||||
monkeypatch,
|
||||
{"license_expression": None, "license": None, "classifiers": []},
|
||||
)
|
||||
checker = _make_checker()
|
||||
assert checker.check_package("mystery-pkg", "1.0.0") is False
|
||||
Reference in New Issue
Block a user