From 4a692eed44e5cea825cd989f9a6f3a609fea747c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 7 Apr 2026 12:16:58 +0200 Subject: [PATCH] News: switched mechanism to fetch news from github This allows news to be updated without updating the installed version of Serena itself. --- .../dashboard/news => news}/20260111.html | 0 .../dashboard/news => news}/20260303.html | 0 .../dashboard/news => news}/20260321.html | 0 .../dashboard/news => news}/20260330.html | 0 .../dashboard/news => news}/20260404.html | 0 news/news.json | 7 ++ scripts/build_news_json.py | 44 ++++++++ src/serena/config/serena_config.py | 12 ++ src/serena/dashboard.py | 103 +++++++++++++++--- src/serena/resources/dashboard/dashboard.js | 79 +++++++------- 10 files changed, 186 insertions(+), 59 deletions(-) rename {src/serena/resources/dashboard/news => news}/20260111.html (100%) rename {src/serena/resources/dashboard/news => news}/20260303.html (100%) rename {src/serena/resources/dashboard/news => news}/20260321.html (100%) rename {src/serena/resources/dashboard/news => news}/20260330.html (100%) rename {src/serena/resources/dashboard/news => news}/20260404.html (100%) create mode 100644 news/news.json create mode 100644 scripts/build_news_json.py diff --git a/src/serena/resources/dashboard/news/20260111.html b/news/20260111.html similarity index 100% rename from src/serena/resources/dashboard/news/20260111.html rename to news/20260111.html diff --git a/src/serena/resources/dashboard/news/20260303.html b/news/20260303.html similarity index 100% rename from src/serena/resources/dashboard/news/20260303.html rename to news/20260303.html diff --git a/src/serena/resources/dashboard/news/20260321.html b/news/20260321.html similarity index 100% rename from src/serena/resources/dashboard/news/20260321.html rename to news/20260321.html diff --git a/src/serena/resources/dashboard/news/20260330.html b/news/20260330.html similarity index 100% rename from src/serena/resources/dashboard/news/20260330.html rename to news/20260330.html diff --git a/src/serena/resources/dashboard/news/20260404.html b/news/20260404.html similarity index 100% rename from src/serena/resources/dashboard/news/20260404.html rename to news/20260404.html diff --git a/news/news.json b/news/news.json new file mode 100644 index 00000000..11eb1d97 --- /dev/null +++ b/news/news.json @@ -0,0 +1,7 @@ +{ + "20260111": "
\n

Extended Symbol Information, Type Hierarchy and Compact Overviews

\n

January 11, 2026

\n

\n Recent commits and a new plugin release provide major new features!\n

\n \n

\n For more detailed information, see our changelog.\n

\n
", + "20260303": "
\n

Nested and Global Memories

\n

March 03, 2026

\n

\n Serena's memory system has been significantly extended in functionality.\n It keeps the simplicity that made it popular but now supports structuring memories\n into topics and sharing them across projects.\n

\n

\n For more detailed information, see the documentation.\n

\n
", + "20260321": "
\n

Progressive Shortening and General Improvements in Tool Responses

\n

March 21, 2026

\n

\n Tools that may deliver too long results now output informative shorter\n messages instead of just stating \"result is too long\". The mechanism uses\n progressive shortening so that the contained information is optimized\n for the length limit.\n

\n

\n In addition, several tool responses have been optimized without losing\n content, making Serena even more token efficient in general.\n

\n
", + "20260330": "
\n

Querying of External Projects

\n

March 30, 2026

\n

\n We added a new tool for querying external projects that are\n known to Serena: From the current project, use Serena's tools on another project to retrieve information.\n This is useful in situations where the current project builds on or is otherwise related\n to other projects.\n

\n

\n Find out more about this feature in our documentation.\n

\n
", + "20260404": "
\n

Serena v1.0 Is Here!

\n

April 3, 2026

\n

\n After a full year of work since the first version of Serena, we are excited to announce the release of Serena v1.0!\n

\n

\n Since our last update, we have now added support for exciting new retrieval and refactoring tools, especially for users of the JetBrains backend.\n

\n \n

\n We are far from done. Many new features will be coming your way – for both the LSP and JetBrains backends.\n

\n

\n Thank you to our contributors, users and customers!\n

\n
" +} \ No newline at end of file diff --git a/scripts/build_news_json.py b/scripts/build_news_json.py new file mode 100644 index 00000000..10a86b67 --- /dev/null +++ b/scripts/build_news_json.py @@ -0,0 +1,44 @@ +""" +Script to build a single JSON file from all individual news HTML files. + +Usage: + uv run python scripts/build_news_json.py + +This reads all .html files from the `news/` directory and creates +`news/news.json` containing a mapping of news IDs to HTML content strings. +""" + +import json +import sys +from pathlib import Path + +from serena.config.serena_config import SerenaPaths + + +def build_news_json() -> None: + news_dir = Path(SerenaPaths().news_dir) + + if not news_dir.exists(): + print(f"Error: News directory not found at {news_dir}", file=sys.stderr) + sys.exit(1) + + news_files = sorted(news_dir.glob("*.html")) + if not news_files: + print("Warning: No HTML news files found in news/", file=sys.stderr) + + news_data: dict[str, str] = {} + for news_file in news_files: + news_id = news_file.stem # e.g. "20260111" + html_content = news_file.read_text(encoding="utf-8").strip() + news_data[news_id] = html_content + print(f" Added news {news_id} ({len(html_content)} chars)") + + output_file = news_dir / "news.json" + with open(output_file, "w", encoding="utf-8") as f: + json.dump(news_data, f, ensure_ascii=False, indent=2) + + print(f"\nBuilt {output_file} with {len(news_data)} news entries.") + + +if __name__ == "__main__": + build_news_json() diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index ae5bbd1d..987d8cf6 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -87,6 +87,18 @@ class SerenaPaths: """ file containing the ID of the last read news snippet """ + self.news_etag_file: str = os.path.join(self.serena_user_home_dir, "news_etag.txt") + """ + file containing the ETag of the last fetched remote news JSON + """ + self.news_file: str = os.path.join(self.serena_user_home_dir, "news.json") + """ + local cache of the remote news JSON file + """ + self.news_dir: str = os.path.join(REPO_ROOT, "news") + """ + repository news directory containing the source HTML snippets and generated news.json + """ global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global")) global_memories_path.mkdir(parents=True, exist_ok=True) self.global_memories_path = global_memories_path diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 11a9739d..dbd46504 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -1,7 +1,10 @@ +import json import os import socket import sys import threading +import urllib.error +import urllib.request from pathlib import Path from typing import TYPE_CHECKING, Any, Self @@ -142,7 +145,11 @@ class SerenaDashboardAPI: self._agent = agent self._app = Flask(__name__) self._tool_usage_stats = tool_usage_stats + self._loaded_news: dict[str, str] = {} + self._news_ready = threading.Event() self._setup_routes() + # Fetch remote news in background on startup (non-blocking) + threading.Thread(target=self._fetch_news, daemon=True).start() @property def memory_log_handler(self) -> MemoryLogHandler: @@ -346,12 +353,12 @@ class SerenaDashboardAPI: except Exception as e: return {"status": "error", "message": str(e)} - @self._app.route("/news_snippet_ids", methods=["GET"]) - def get_news_snippet_ids() -> dict[str, str | list[int]]: - def _get_unread_news_ids() -> list[int]: - all_news_files = (Path(SERENA_DASHBOARD_DIR) / "news").glob("*.html") - all_news_ids = [int(f.stem) for f in all_news_files] - """News ids are ints of format YYYYMMDD (publication dates)""" + @self._app.route("/fetch_unread_news", methods=["GET"]) + def fetch_unread_news() -> dict[str, dict[str, str] | str]: + def _fetch_unread_news() -> dict[str, str]: + """News ids are strings of format YYYYMMDD (publication dates)""" + self._news_ready.wait() + all_news = self._loaded_news # Filter news items by installation date serena_config_creation_date = SerenaConfig.get_config_file_creation_date() @@ -359,23 +366,23 @@ class SerenaDashboardAPI: # should not normally happen, since config file should exist when the dashboard is started # We assume a fresh installation in this case log.error("Serena config file not found when starting the dashboard") - return [] - serena_config_creation_date_int = int(serena_config_creation_date.strftime("%Y%m%d")) + return {} + serena_config_creation_date = serena_config_creation_date.strftime("%Y%m%d") # Only include news items published on or after the installation date - post_installation_news_ids = [news_id for news_id in all_news_ids if news_id >= serena_config_creation_date_int] + post_installation_news = {k: v for k, v in all_news.items() if k >= serena_config_creation_date} news_snippet_id_file = SerenaPaths().news_snippet_id_file if not os.path.exists(news_snippet_id_file): - return post_installation_news_ids + return post_installation_news with open(news_snippet_id_file, encoding="utf-8") as f: - last_read_news_id = int(f.read().strip()) - if last_read_news_id == 20262103: - last_read_news_id = 20260321 # fix originally misnamed file - return [news_id for news_id in post_installation_news_ids if news_id > last_read_news_id] + last_read_news_id = f.read().strip() + if last_read_news_id == "20262103": + last_read_news_id = "20260321" # fix originally misnamed news id + return {k: v for k, v in post_installation_news.items() if k > last_read_news_id} try: - unread_news_ids = _get_unread_news_ids() - return {"news_snippet_ids": unread_news_ids, "status": "success"} + unread_news = _fetch_unread_news() + return {"news": unread_news, "status": "success"} except Exception as e: return {"status": "error", "message": str(e)} @@ -383,10 +390,10 @@ class SerenaDashboardAPI: def mark_news_snippet_as_read() -> dict[str, str]: try: request_data = request.get_json() - news_snippet_id = int(request_data.get("news_snippet_id")) + news_snippet_id = str(request_data.get("news_snippet_id")) news_snippet_id_file = SerenaPaths().news_snippet_id_file with open(news_snippet_id_file, "w", encoding="utf-8") as f: - f.write(str(news_snippet_id)) + f.write(news_snippet_id) return {"status": "success", "message": f"Marked news snippet {news_snippet_id} as read"} except Exception as e: return {"status": "error", "message": str(e)} @@ -618,6 +625,66 @@ class SerenaDashboardAPI: self._agent.execute_task(run, logged=True, name="SaveSerenaConfig") + # ===== Remote News Methods ===== + + # The branch from which news are fetched. Change to a feature branch for testing. + _NEWS_JSON_URL = "https://raw.githubusercontent.com/oraios/serena/main/news/news.json" + + def _fetch_news(self) -> None: + """Fetch news.json from GitHub using ETag-based caching and store in memory. Silently ignores network errors.""" + paths = SerenaPaths() + + headers: dict[str, str] = {} + # Load stored ETag if available + if os.path.exists(paths.news_etag_file) and os.path.exists(paths.news_file): + try: + with open(paths.news_etag_file, encoding="utf-8") as f: + stored_etag = f.read().strip() + if stored_etag: + headers["If-None-Match"] = stored_etag + except Exception: + log.warning("Failed to read stored news ETag at %s, proceeding without it", paths.news_etag_file, exc_info=True) + + fetched_news_dict = None + try: + req = urllib.request.Request(self._NEWS_JSON_URL, headers=headers) + with urllib.request.urlopen(req, timeout=10) as response: + etag = response.headers.get("ETag", "") + body = response.read().decode("utf-8") + # Validate JSON + fetched_news_dict = json.loads(body) + # Store news content and ETag + with open(paths.news_file, "w", encoding="utf-8") as f: + f.write(body) + if etag: + with open(paths.news_etag_file, "w", encoding="utf-8") as f: + f.write(etag) + log.info("Remote news updated from %s", self._NEWS_JSON_URL) + except urllib.error.HTTPError as e: + if e.code == 304: + log.debug("Remote news unchanged (304 Not Modified)") + else: + log.warning("Failed to fetch remote news (HTTP %d): %s", e.code, e.reason) + except Exception as e: + log.warning("Failed to fetch remote news: %s", e) + if fetched_news_dict is None: + fetched_news_dict = self._load_previously_fetched_news_data() + self._loaded_news = fetched_news_dict + self._news_ready.set() + + @staticmethod + def _load_previously_fetched_news_data() -> dict[str, str]: + """Return the news data dict. Uses local cache if available, otherwise falls back to local news files.""" + paths = SerenaPaths() + + if os.path.exists(paths.news_file): + try: + with open(paths.news_file, encoding="utf-8") as f: + return json.loads(f.read()) + except Exception: + log.warning("Failed to read cached news data from %s", paths.news_file) + return {} + def _add_language(self, request_add_language: RequestAddLanguage) -> None: from solidlsp.ls_config import Language diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index dffd10ef..784d8238 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -2075,31 +2075,36 @@ class Dashboard { let self = this; console.log('Loading news...'); $.ajax({ - url: '/news_snippet_ids', + url: '/fetch_unread_news', type: 'GET', success: function(response) { - console.log('News snippet IDs response:', response); - if (response.status === 'success' && response.news_snippet_ids && response.news_snippet_ids.length > 0) { - console.log('Displaying news with IDs:', response.news_snippet_ids); - self.displayNews(response.news_snippet_ids); + console.log('Unread news response:', response); + if (response.status === 'success' && response.news && Object.keys(response.news).length > 0) { + const newsIds = Object.keys(response.news); + self.displayNews(newsIds, response.news); } else { console.log('No unread news, hiding section'); self.$newsSection.hide(); } }, error: function(xhr, status, error) { - console.error('Error loading news snippet IDs:', error); + console.error('Error loading news:', error); self.$newsSection.hide(); } }); } - displayNews(newsIds) { + /** + * Display news items given unread IDs and the full news data mapping. + * @param {number[]} newsIds - array of unread news IDs + * @param {Object} newsData - mapping of news ID strings to HTML content + */ + displayNews(newsIds, newsData) { let self = this; console.log('displayNews called with:', newsIds); // Sort newest first (descending order) newsIds.sort((a, b) => b - a); - + if (newsIds.length === 0) { console.log('No news items to display.'); self.$newsSection.hide(); @@ -2108,40 +2113,32 @@ class Dashboard { self.$newsSection.show(); self.$newsDisplay.empty(); console.log('Displaying ' + newsIds.length + ' news items.'); - // Load each news snippet HTML - let loadedCount = 0; - newsIds.forEach(function(newsId) { - $.ajax({ - url: '/dashboard/news/' + newsId + '.html', - type: 'GET', - success: function(html) { - // Wrap the HTML in a container with a button - let $newsContainer = $('
').attr('data-news-id', newsId); - let $newsContent = $(html); - - // Add button for marking as read - let $markRead = $('
'); - let $button = $('