feat: fetch blog posts from docs RSS feed instead of static JSON on GitHub

This commit is contained in:
Ryan Crabbe
2026-03-16 15:55:16 -07:00
parent 4f2fe3378f
commit 67482db49d
3 changed files with 122 additions and 51 deletions
+1 -1
View File
@@ -358,7 +358,7 @@ model_cost_map_url: str = os.getenv(
)
blog_posts_url: str = os.getenv(
"LITELLM_BLOG_POSTS_URL",
"https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/blog_posts.json",
"https://docs.litellm.ai/blog/rss.xml",
)
anthropic_beta_headers_url: str = os.getenv(
"LITELLM_ANTHROPIC_BETA_HEADERS_URL",
+60 -22
View File
@@ -1,8 +1,8 @@
"""
Pulls the latest LiteLLM blog posts from GitHub.
Pulls the latest LiteLLM blog posts from the docs RSS feed.
Falls back to the bundled local backup on any failure.
GitHub JSON URL is configured via litellm.blog_posts_url (or LITELLM_BLOG_POSTS_URL env var).
RSS URL is configured via litellm.blog_posts_url (or LITELLM_BLOG_POSTS_URL env var).
Disable remote fetching entirely:
export LITELLM_LOCAL_BLOG_POSTS=True
@@ -11,6 +11,8 @@ Disable remote fetching entirely:
import json
import os
import time
import xml.etree.ElementTree as ET
from email.utils import parsedate_to_datetime
from importlib.resources import files
from typing import Any, Dict, List, Optional
@@ -37,9 +39,8 @@ class GetBlogPosts:
"""
Fetches, validates, and caches LiteLLM blog posts.
Mirrors the structure of GetModelCostMap:
- Fetches from GitHub with a 5-second timeout
- Validates the response has a non-empty ``posts`` list
- Fetches RSS feed from docs site with a 5-second timeout
- Parses the XML and extracts the latest blog post
- Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour)
- Falls back to the bundled local backup on any failure
"""
@@ -56,30 +57,67 @@ class GetBlogPosts:
return content.get("posts", [])
@staticmethod
def fetch_remote_blog_posts(url: str, timeout: int = 5) -> dict:
def fetch_rss_feed(url: str, timeout: int = 5) -> str:
"""
Fetch blog posts JSON from a remote URL.
Fetch RSS XML from a remote URL.
Returns the parsed response. Raises on network/parse errors.
Returns the raw XML text. Raises on network errors.
"""
response = httpx.get(url, timeout=timeout)
response.raise_for_status()
return response.json()
return response.text
@staticmethod
def validate_blog_posts(data: Any) -> bool:
"""Return True if data is a dict with a non-empty ``posts`` list."""
if not isinstance(data, dict):
verbose_logger.warning(
"LiteLLM: Blog posts response is not a dict (type=%s). "
"Falling back to local backup.",
type(data).__name__,
def parse_rss_to_posts(xml_text: str, max_posts: int = 1) -> List[Dict[str, str]]:
"""
Parse RSS XML and return a list of blog post dicts.
Extracts title, description, date (YYYY-MM-DD), and url from each <item>.
"""
root = ET.fromstring(xml_text)
channel = root.find("channel")
if channel is None:
raise ValueError("RSS feed missing <channel> element")
posts: List[Dict[str, str]] = []
for item in channel.findall("item"):
if len(posts) >= max_posts:
break
title_el = item.find("title")
link_el = item.find("link")
desc_el = item.find("description")
pub_date_el = item.find("pubDate")
if title_el is None or link_el is None:
continue
# Parse RFC 2822 date to YYYY-MM-DD
date_str = ""
if pub_date_el is not None and pub_date_el.text:
try:
dt = parsedate_to_datetime(pub_date_el.text)
date_str = dt.strftime("%Y-%m-%d")
except Exception:
date_str = pub_date_el.text
posts.append(
{
"title": title_el.text or "",
"description": desc_el.text or "" if desc_el is not None else "",
"date": date_str,
"url": link_el.text or "",
}
)
return False
posts = data.get("posts")
return posts
@staticmethod
def validate_blog_posts(posts: List[Dict[str, str]]) -> bool:
"""Return True if posts is a non-empty list."""
if not isinstance(posts, list) or len(posts) == 0:
verbose_logger.warning(
"LiteLLM: Blog posts response has no valid 'posts' list. "
"LiteLLM: Parsed RSS feed has no valid posts. "
"Falling back to local backup.",
)
return False
@@ -102,7 +140,8 @@ class GetBlogPosts:
return cached
try:
data = cls.fetch_remote_blog_posts(url)
xml_text = cls.fetch_rss_feed(url)
posts = cls.parse_rss_to_posts(xml_text)
except Exception as e:
verbose_logger.warning(
"LiteLLM: Failed to fetch blog posts from %s: %s. "
@@ -112,10 +151,9 @@ class GetBlogPosts:
)
return cls.load_local_blog_posts()
if not cls.validate_blog_posts(data):
if not cls.validate_blog_posts(posts):
return cls.load_local_blog_posts()
posts = data["posts"]
cls._cached_posts = posts
cls._last_fetch_time = now
return posts
+61 -28
View File
@@ -1,5 +1,4 @@
"""Tests for GetBlogPosts utility class."""
import json
import time
from unittest.mock import MagicMock, patch
@@ -13,16 +12,26 @@ from litellm.litellm_core_utils.get_blog_posts import (
get_blog_posts,
)
SAMPLE_RESPONSE = {
"posts": [
{
"title": "Test Post",
"description": "A test post.",
"date": "2026-01-01",
"url": "https://www.litellm.ai/blog/test",
}
]
}
SAMPLE_RSS = """\
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>LiteLLM Blog</title>
<item>
<title>Test Post</title>
<link>https://docs.litellm.ai/blog/test</link>
<description>A test post.</description>
<pubDate>Wed, 01 Jan 2026 10:00:00 GMT</pubDate>
</item>
<item>
<title>Second Post</title>
<link>https://docs.litellm.ai/blog/second</link>
<description>Another post.</description>
<pubDate>Tue, 31 Dec 2025 10:00:00 GMT</pubDate>
</item>
</channel>
</rss>
"""
@pytest.fixture(autouse=True)
@@ -45,26 +54,48 @@ def test_load_local_blog_posts_returns_list():
assert "url" in first
def test_parse_rss_to_posts():
posts = GetBlogPosts.parse_rss_to_posts(SAMPLE_RSS, max_posts=1)
assert len(posts) == 1
assert posts[0]["title"] == "Test Post"
assert posts[0]["url"] == "https://docs.litellm.ai/blog/test"
assert posts[0]["description"] == "A test post."
assert posts[0]["date"] == "2026-01-01"
def test_parse_rss_to_posts_multiple():
posts = GetBlogPosts.parse_rss_to_posts(SAMPLE_RSS, max_posts=5)
assert len(posts) == 2
assert posts[1]["title"] == "Second Post"
def test_parse_rss_to_posts_invalid_xml():
with pytest.raises(Exception):
GetBlogPosts.parse_rss_to_posts("not xml")
def test_parse_rss_to_posts_missing_channel():
with pytest.raises(ValueError, match="missing <channel>"):
GetBlogPosts.parse_rss_to_posts("<rss></rss>")
def test_validate_blog_posts_valid():
assert GetBlogPosts.validate_blog_posts(SAMPLE_RESPONSE) is True
def test_validate_blog_posts_missing_posts_key():
assert GetBlogPosts.validate_blog_posts({"other": []}) is False
posts = [{"title": "T", "description": "D", "date": "2026-01-01", "url": "https://x.com"}]
assert GetBlogPosts.validate_blog_posts(posts) is True
def test_validate_blog_posts_empty_list():
assert GetBlogPosts.validate_blog_posts({"posts": []}) is False
assert GetBlogPosts.validate_blog_posts([]) is False
def test_validate_blog_posts_not_dict():
assert GetBlogPosts.validate_blog_posts("not a dict") is False
def test_validate_blog_posts_not_list():
assert GetBlogPosts.validate_blog_posts("not a list") is False
def test_get_blog_posts_success():
"""Fetches from remote on first call."""
"""Fetches from RSS on first call."""
mock_response = MagicMock()
mock_response.json.return_value = SAMPLE_RESPONSE
mock_response.text = SAMPLE_RSS
mock_response.raise_for_status = MagicMock()
with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", return_value=mock_response):
@@ -86,10 +117,10 @@ def test_get_blog_posts_network_error_falls_back_to_local():
assert len(posts) > 0
def test_get_blog_posts_invalid_json_falls_back_to_local():
"""Falls back when remote returns non-dict."""
def test_get_blog_posts_invalid_xml_falls_back_to_local():
"""Falls back when remote returns invalid XML."""
mock_response = MagicMock()
mock_response.json.return_value = "not a dict"
mock_response.text = "not valid xml"
mock_response.raise_for_status = MagicMock()
with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get", return_value=mock_response):
@@ -101,7 +132,8 @@ def test_get_blog_posts_invalid_json_falls_back_to_local():
def test_get_blog_posts_ttl_cache_not_refetched():
"""Within TTL window, does not re-fetch."""
GetBlogPosts._cached_posts = SAMPLE_RESPONSE["posts"]
cached = [{"title": "Cached", "description": "D", "date": "2026-01-01", "url": "https://x.com"}]
GetBlogPosts._cached_posts = cached
GetBlogPosts._last_fetch_time = time.time() # just now
call_count = 0
@@ -110,7 +142,7 @@ def test_get_blog_posts_ttl_cache_not_refetched():
nonlocal call_count
call_count += 1
m = MagicMock()
m.json.return_value = SAMPLE_RESPONSE
m.text = SAMPLE_RSS
m.raise_for_status = MagicMock()
return m
@@ -123,11 +155,12 @@ def test_get_blog_posts_ttl_cache_not_refetched():
def test_get_blog_posts_ttl_expired_refetches():
"""After TTL window, re-fetches from remote."""
GetBlogPosts._cached_posts = SAMPLE_RESPONSE["posts"]
cached = [{"title": "Cached", "description": "D", "date": "2026-01-01", "url": "https://x.com"}]
GetBlogPosts._cached_posts = cached
GetBlogPosts._last_fetch_time = time.time() - 7200 # 2 hours ago
mock_response = MagicMock()
mock_response.json.return_value = SAMPLE_RESPONSE
mock_response.text = SAMPLE_RSS
mock_response.raise_for_status = MagicMock()
with patch(