News: switched mechanism to fetch news from github

This allows news to be updated without updating the installed version of Serena itself.

news/news.json

@@ -0,0 +1,7 @@
+{
+  "20260111": "<div class=\"news-item\">\n    <h3>Extended Symbol Information, Type Hierarchy and Compact Overviews</h3>\n    <p class=\"date\">January 11, 2026</p>\n    <p>\n        Recent commits and a new plugin release provide major new features!\n    </p>\n    <ul>\n        <li><strong>Extended symbol information:</strong> The `find_symbol` and `find_referencing_symbols` tools (and their JetBrains variants)\n            now can return more information about the symbol (specifically, docstrings and signatures).\n        </li>\n        <li><strong>Type hierarchy tool:</strong> A new tool &ndash; exclusive to <a href=\"https://oraios.github.io/serena/02-usage/025_jetbrains_plugin.html\">JetBrains mode</a> &ndash;\n            that can fetch a symbol's hierarchy, i.e. superclasses and subclasses, which is very useful in many situations.\n        </li>\n        <li><strong>Compact overviews:</strong> The `get_symbols_overview` tools now return a much more compact\n            representation, saving many tokens. The JetBrains variant of the tool can now return a file's docstring (LSP\n            varian't can't do that yet).\n        </li>\n    </ul>\n    <p>\n        For more detailed information, see our <a href=\"https://github.com/oraios/serena/blob/main/CHANGELOG.md\">changelog</a>.\n    </p>\n</div>",
+  "20260303": "<div class=\"news-item\">\n    <h3>Nested and Global Memories</h3>\n    <p class=\"date\">March 03, 2026</p>\n    <p>\n        Serena's memory system has been significantly extended in functionality.\n        It keeps the simplicity that made it popular but now supports structuring memories\n        into topics and sharing them across projects.\n    </p>\n    <p>\n        For more detailed information, see the <a href=\"https://oraios.github.io/serena/02-usage/045_memories.html\">documentation</a>.\n    </p>\n</div>",
+  "20260321": "<div class=\"news-item\">\n    <h3>Progressive Shortening and General Improvements in Tool Responses</h3>\n    <p class=\"date\">March 21, 2026</p>\n    <p>\n        Tools that may deliver too long results now output informative shorter\n        messages instead of just stating \"result is too long\". The mechanism uses\n        progressive shortening so that the contained information is optimized\n        for the length limit.\n    </p>\n    <p>\n        In addition, several tool responses have been optimized without losing\n        content, making Serena even more token efficient in general.\n    </p>\n</div>",
+  "20260330": "<div class=\"news-item\">\n    <h3>Querying of External Projects</h3>\n    <p class=\"date\">March 30, 2026</p>\n    <p>\n        We added a new tool for querying external projects that are\n        known to Serena: From the current project, use Serena's tools on another project to retrieve information.\n        This is useful in situations where the current project builds on or is otherwise related\n        to other projects.\n    </p>\n    <p>\n        Find out more about this feature in our <a href=\"https://oraios.github.io/serena/02-usage/040_workflow.html#reading-from-external-projects\">documentation</a>.\n    </p>\n</div>",
+  "20260404": "<div class=\"news-item\">\n    <h3>Serena v1.0 Is Here!</h3>\n    <p class=\"date\">April 3, 2026</p>\n    <p>\n        After a full year of work since the first version of Serena, we are excited to announce the release of Serena v1.0!\n    </p>\n    <p>\n        Since our last update, we have now added support for <b>exciting new retrieval and refactoring tools</b>, especially for users of the JetBrains backend.\n    </p>\n    <ul>\n        <li>safe_delete, jet_brains_safe_delete (beta):<br> Safely deletes a symbol, checking for remaining usages first</li>\n        <li>jet_brains_move (beta):<br> Moves a symbol, file or directory to a new location, updating all references</li>\n        <li>jet_brains_find_declaration:<br> Finds the declaration of a symbol using the JetBrains backend</li>\n        <li>jet_brains_find_implementations:<br> Finds the implementations of a symbol (e.g. abstract method)</li>\n        <li>jet_brains_inline_symbol (beta):<br> Inlines a symbol, replacing all call sites with the symbol’s body</li>\n    </ul>\n    <p>\n        We are far from done. Many new features will be coming your way – for both the LSP and JetBrains backends.\n    </p>\n    <p>\n        Thank you to our contributors, users and customers!\n    </p>\n</div>"
+}

scripts/build_news_json.py

@@ -0,0 +1,44 @@
+"""
+Script to build a single JSON file from all individual news HTML files.
+
+Usage:
+    uv run python scripts/build_news_json.py
+
+This reads all .html files from the `news/` directory and creates
+`news/news.json` containing a mapping of news IDs to HTML content strings.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+from serena.config.serena_config import SerenaPaths
+
+
+def build_news_json() -> None:
+    news_dir = Path(SerenaPaths().news_dir)
+
+    if not news_dir.exists():
+        print(f"Error: News directory not found at {news_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    news_files = sorted(news_dir.glob("*.html"))
+    if not news_files:
+        print("Warning: No HTML news files found in news/", file=sys.stderr)
+
+    news_data: dict[str, str] = {}
+    for news_file in news_files:
+        news_id = news_file.stem  # e.g. "20260111"
+        html_content = news_file.read_text(encoding="utf-8").strip()
+        news_data[news_id] = html_content
+        print(f"  Added news {news_id} ({len(html_content)} chars)")
+
+    output_file = news_dir / "news.json"
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(news_data, f, ensure_ascii=False, indent=2)
+
+    print(f"\nBuilt {output_file} with {len(news_data)} news entries.")
+
+
+if __name__ == "__main__":
+    build_news_json()

src/serena/config/serena_config.py

@@ -87,6 +87,18 @@ class SerenaPaths:
         """
         file containing the ID of the last read news snippet
         """
+        self.news_etag_file: str = os.path.join(self.serena_user_home_dir, "news_etag.txt")
+        """
+        file containing the ETag of the last fetched remote news JSON
+        """
+        self.news_file: str = os.path.join(self.serena_user_home_dir, "news.json")
+        """
+        local cache of the remote news JSON file
+        """
+        self.news_dir: str = os.path.join(REPO_ROOT, "news")
+        """
+        repository news directory containing the source HTML snippets and generated news.json
+        """
         global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global"))
         global_memories_path.mkdir(parents=True, exist_ok=True)
         self.global_memories_path = global_memories_path

src/serena/dashboard.py

@@ -1,7 +1,10 @@
+import json
 import os
 import socket
 import sys
 import threading
+import urllib.error
+import urllib.request
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Self
 
@@ -142,7 +145,11 @@ class SerenaDashboardAPI:
         self._agent = agent
         self._app = Flask(__name__)
         self._tool_usage_stats = tool_usage_stats
+        self._loaded_news: dict[str, str] = {}
+        self._news_ready = threading.Event()
         self._setup_routes()
+        # Fetch remote news in background on startup (non-blocking)
+        threading.Thread(target=self._fetch_news, daemon=True).start()
 
     @property
     def memory_log_handler(self) -> MemoryLogHandler:
@@ -346,12 +353,12 @@ class SerenaDashboardAPI:
             except Exception as e:
                 return {"status": "error", "message": str(e)}
 
-        @self._app.route("/news_snippet_ids", methods=["GET"])
-        def get_news_snippet_ids() -> dict[str, str | list[int]]:
-            def _get_unread_news_ids() -> list[int]:
-                all_news_files = (Path(SERENA_DASHBOARD_DIR) / "news").glob("*.html")
-                all_news_ids = [int(f.stem) for f in all_news_files]
-                """News ids are ints of format YYYYMMDD (publication dates)"""
+        @self._app.route("/fetch_unread_news", methods=["GET"])
+        def fetch_unread_news() -> dict[str, dict[str, str] | str]:
+            def _fetch_unread_news() -> dict[str, str]:
+                """News ids are strings of format YYYYMMDD (publication dates)"""
+                self._news_ready.wait()
+                all_news = self._loaded_news
 
                 # Filter news items by installation date
                 serena_config_creation_date = SerenaConfig.get_config_file_creation_date()
@@ -359,23 +366,23 @@ class SerenaDashboardAPI:
                     # should not normally happen, since config file should exist when the dashboard is started
                     # We assume a fresh installation in this case
                     log.error("Serena config file not found when starting the dashboard")
-                    return []
-                serena_config_creation_date_int = int(serena_config_creation_date.strftime("%Y%m%d"))
+                    return {}
+                serena_config_creation_date = serena_config_creation_date.strftime("%Y%m%d")
                 # Only include news items published on or after the installation date
-                post_installation_news_ids = [news_id for news_id in all_news_ids if news_id >= serena_config_creation_date_int]
+                post_installation_news = {k: v for k, v in all_news.items() if k >= serena_config_creation_date}
 
                 news_snippet_id_file = SerenaPaths().news_snippet_id_file
                 if not os.path.exists(news_snippet_id_file):
-                    return post_installation_news_ids
+                    return post_installation_news
                 with open(news_snippet_id_file, encoding="utf-8") as f:
-                    last_read_news_id = int(f.read().strip())
-                    if last_read_news_id == 20262103:
-                        last_read_news_id = 20260321  # fix originally misnamed file
-                return [news_id for news_id in post_installation_news_ids if news_id > last_read_news_id]
+                    last_read_news_id = f.read().strip()
+                    if last_read_news_id == "20262103":
+                        last_read_news_id = "20260321"  # fix originally misnamed news id
+                return {k: v for k, v in post_installation_news.items() if k > last_read_news_id}
 
             try:
-                unread_news_ids = _get_unread_news_ids()
-                return {"news_snippet_ids": unread_news_ids, "status": "success"}
+                unread_news = _fetch_unread_news()
+                return {"news": unread_news, "status": "success"}
             except Exception as e:
                 return {"status": "error", "message": str(e)}
 
@@ -383,10 +390,10 @@ class SerenaDashboardAPI:
         def mark_news_snippet_as_read() -> dict[str, str]:
             try:
                 request_data = request.get_json()
-                news_snippet_id = int(request_data.get("news_snippet_id"))
+                news_snippet_id = str(request_data.get("news_snippet_id"))
                 news_snippet_id_file = SerenaPaths().news_snippet_id_file
                 with open(news_snippet_id_file, "w", encoding="utf-8") as f:
-                    f.write(str(news_snippet_id))
+                    f.write(news_snippet_id)
                 return {"status": "success", "message": f"Marked news snippet {news_snippet_id} as read"}
             except Exception as e:
                 return {"status": "error", "message": str(e)}
@@ -618,6 +625,66 @@ class SerenaDashboardAPI:
 
         self._agent.execute_task(run, logged=True, name="SaveSerenaConfig")
 
+    # ===== Remote News Methods =====
+
+    # The branch from which news are fetched. Change to a feature branch for testing.
+    _NEWS_JSON_URL = "https://raw.githubusercontent.com/oraios/serena/main/news/news.json"
+
+    def _fetch_news(self) -> None:
+        """Fetch news.json from GitHub using ETag-based caching and store in memory. Silently ignores network errors."""
+        paths = SerenaPaths()
+
+        headers: dict[str, str] = {}
+        # Load stored ETag if available
+        if os.path.exists(paths.news_etag_file) and os.path.exists(paths.news_file):
+            try:
+                with open(paths.news_etag_file, encoding="utf-8") as f:
+                    stored_etag = f.read().strip()
+                if stored_etag:
+                    headers["If-None-Match"] = stored_etag
+            except Exception:
+                log.warning("Failed to read stored news ETag at %s, proceeding without it", paths.news_etag_file, exc_info=True)
+
+        fetched_news_dict = None
+        try:
+            req = urllib.request.Request(self._NEWS_JSON_URL, headers=headers)
+            with urllib.request.urlopen(req, timeout=10) as response:
+                etag = response.headers.get("ETag", "")
+                body = response.read().decode("utf-8")
+                # Validate JSON
+                fetched_news_dict = json.loads(body)
+                # Store news content and ETag
+                with open(paths.news_file, "w", encoding="utf-8") as f:
+                    f.write(body)
+                if etag:
+                    with open(paths.news_etag_file, "w", encoding="utf-8") as f:
+                        f.write(etag)
+                log.info("Remote news updated from %s", self._NEWS_JSON_URL)
+        except urllib.error.HTTPError as e:
+            if e.code == 304:
+                log.debug("Remote news unchanged (304 Not Modified)")
+            else:
+                log.warning("Failed to fetch remote news (HTTP %d): %s", e.code, e.reason)
+        except Exception as e:
+            log.warning("Failed to fetch remote news: %s", e)
+        if fetched_news_dict is None:
+            fetched_news_dict = self._load_previously_fetched_news_data()
+        self._loaded_news = fetched_news_dict
+        self._news_ready.set()
+
+    @staticmethod
+    def _load_previously_fetched_news_data() -> dict[str, str]:
+        """Return the news data dict. Uses local cache if available, otherwise falls back to local news files."""
+        paths = SerenaPaths()
+
+        if os.path.exists(paths.news_file):
+            try:
+                with open(paths.news_file, encoding="utf-8") as f:
+                    return json.loads(f.read())
+            except Exception:
+                log.warning("Failed to read cached news data from %s", paths.news_file)
+        return {}
+
     def _add_language(self, request_add_language: RequestAddLanguage) -> None:
         from solidlsp.ls_config import Language
 

src/serena/resources/dashboard/dashboard.js

@@ -2075,31 +2075,36 @@ class Dashboard {
         let self = this;
         console.log('Loading news...');
         $.ajax({
-            url: '/news_snippet_ids',
+            url: '/fetch_unread_news',
             type: 'GET',
             success: function(response) {
-                console.log('News snippet IDs response:', response);
-                if (response.status === 'success' && response.news_snippet_ids && response.news_snippet_ids.length > 0) {
-                    console.log('Displaying news with IDs:', response.news_snippet_ids);
-                    self.displayNews(response.news_snippet_ids);
+                console.log('Unread news response:', response);
+                if (response.status === 'success' && response.news && Object.keys(response.news).length > 0) {
+                    const newsIds = Object.keys(response.news);
+                    self.displayNews(newsIds, response.news);
                 } else {
                     console.log('No unread news, hiding section');
                     self.$newsSection.hide();
                 }
             },
             error: function(xhr, status, error) {
-                console.error('Error loading news snippet IDs:', error);
+                console.error('Error loading news:', error);
                 self.$newsSection.hide();
             }
         });
     }
 
-    displayNews(newsIds) {
+    /**
+     * Display news items given unread IDs and the full news data mapping.
+     * @param {number[]} newsIds - array of unread news IDs
+     * @param {Object} newsData - mapping of news ID strings to HTML content
+     */
+    displayNews(newsIds, newsData) {
         let self = this;
         console.log('displayNews called with:', newsIds);
         // Sort newest first (descending order)
         newsIds.sort((a, b) => b - a);
-        
+
         if (newsIds.length === 0) {
             console.log('No news items to display.');
             self.$newsSection.hide();
@@ -2108,40 +2113,32 @@ class Dashboard {
         self.$newsSection.show();
         self.$newsDisplay.empty();
         console.log('Displaying ' + newsIds.length + ' news items.');
-        // Load each news snippet HTML
-        let loadedCount = 0;
-        newsIds.forEach(function(newsId) {
-            $.ajax({
-                url: '/dashboard/news/' + newsId + '.html',
-                type: 'GET',
-                success: function(html) {
-                    // Wrap the HTML in a container with a button
-                    let $newsContainer = $('<div class="news-container">').attr('data-news-id', newsId);
-                    let $newsContent = $(html);
-                    
-                    // Add button for marking as read
-                    let $markRead = $('<div class="news-mark-read">');
-                    let $button = $('<button class="news-mark-read-btn">').attr('data-news-id', newsId).text('Mark as read');
 
-                    $markRead.append($button);
-                    $newsContent.append($markRead);
-                    
-                    $newsContainer.append($newsContent);
-                    self.$newsDisplay.append($newsContainer);
-                    
-                    // Bind button click event
-                    $button.on('click', function() {
-                        const btn = $(this);
-                        btn.prop('disabled', true).text('Marking...');
-                        self.markNewsAsRead(newsId);
-                    });
-                    
-                    loadedCount++;
-                },
-                error: function(xhr, status, error) {
-                    console.error('Error loading news snippet ' + newsId + ':', error);
-                    loadedCount++;
-                }
+        newsIds.forEach(function(newsId) {
+            const html = newsData[String(newsId)];
+            if (!html) {
+                console.warn('No news content found for ID ' + newsId);
+                return;
+            }
+            // Wrap the HTML in a container with a button
+            let $newsContainer = $('<div class="news-container">').attr('data-news-id', newsId);
+            let $newsContent = $(html);
+
+            // Add button for marking as read
+            let $markRead = $('<div class="news-mark-read">');
+            let $button = $('<button class="news-mark-read-btn">').attr('data-news-id', newsId).text('Mark as read');
+
+            $markRead.append($button);
+            $newsContent.append($markRead);
+
+            $newsContainer.append($newsContent);
+            self.$newsDisplay.append($newsContainer);
+
+            // Bind button click event
+            $button.on('click', function() {
+                const btn = $(this);
+                btn.prop('disabled', true).text('Marking...');
+                self.markNewsAsRead(newsId);
             });
         });
     }