Merge pull request #321 from oraios/capture-startup-logs

Capture startup logs and other logging improvements

pyproject.toml

@@ -235,6 +235,7 @@ ignore = [
   "RUF012",   # forbids mutable attributes as ClassVar
   "SIM117",   # forbids nested with statements
   "C400",     # wants to unnecessarily force use of list comprehension
+  "UP037",    # can incorrectly (!) convert quoted type to unquoted type, causing an error
 ]
 unfixable = ["F841", "F601", "F602", "B018"]
 extend-fixable = ["F401", "B905", "W291"]

src/serena/agent.py

@@ -13,7 +13,7 @@ from collections.abc import Callable
 from concurrent.futures import Future, ThreadPoolExecutor
 from logging import Logger
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 from sensai.util import logging
 from sensai.util.logging import LogTime
@@ -22,17 +22,15 @@ from serena import serena_version
 from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode
 from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir
-from serena.constants import (
-    SERENA_LOG_FORMAT,
-)
-from serena.dashboard import MemoryLogHandler, SerenaDashboardAPI
+from serena.dashboard import SerenaDashboardAPI
 from serena.project import Project
 from serena.prompt_factory import SerenaPromptFactory
 from serena.tools import ActivateProjectTool, Tool, ToolRegistry
+from serena.util.logging import MemoryLogHandler
 from solidlsp import SolidLanguageServer
 
 if TYPE_CHECKING:
-    from serena.gui_log_viewer import GuiLogViewerHandler
+    from serena.gui_log_viewer import GuiLogViewer
 
 log = logging.getLogger(__name__)
 TTool = TypeVar("TTool", bound="Tool")
@@ -101,6 +99,7 @@ class SerenaAgent:
         serena_config: SerenaConfig | None = None,
         context: SerenaAgentContext | None = None,
         modes: list[SerenaAgentMode] | None = None,
+        memory_log_handler: MemoryLogHandler | None = None,
     ):
         """
         :param project: the project to load immediately or None to not load any project; may be a path to the project or a name of
@@ -111,6 +110,8 @@ class SerenaAgent:
             The context may adjust prompts, tool availability, and tool descriptions.
         :param modes: list of modes in which the agent is operating (they will be combined), None for default modes.
             The modes may adjust prompts, tool availability, and tool descriptions.
+        :param memory_log_handler: a MemoryLogHandler instance from which to read log messages; if None, a new one will be created
+            if necessary.
         """
         # obtain serena configuration using the decoupled factory function
         self.serena_config = serena_config or SerenaConfig.from_config_file()
@@ -121,20 +122,25 @@ class SerenaAgent:
             log.info(f"Changing the root logger level to {serena_log_level}")
             Logger.root.setLevel(serena_log_level)
 
+        def get_memory_log_handler() -> MemoryLogHandler:
+            nonlocal memory_log_handler
+            if memory_log_handler is None:
+                memory_log_handler = MemoryLogHandler(level=serena_log_level)
+                Logger.root.addHandler(memory_log_handler)
+            return memory_log_handler
+
         # open GUI log window if enabled
-        self._gui_log_handler: Union["GuiLogViewerHandler", None] = None  # noqa
+        self._gui_log_viewer: Optional["GuiLogViewer"] = None
         if self.serena_config.gui_log_window_enabled:
             if platform.system() == "Darwin":
                 log.warning("GUI log window is not supported on macOS")
             else:
                 # even importing on macOS may fail if tkinter dependencies are unavailable (depends on Python interpreter installation
                 # which uv used as a base, unfortunately)
-                from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler
+                from serena.gui_log_viewer import GuiLogViewer
 
-                self._gui_log_handler = GuiLogViewerHandler(
-                    GuiLogViewer("dashboard", title="Serena Logs"), level=serena_log_level, format_string=SERENA_LOG_FORMAT
-                )
-                Logger.root.addHandler(self._gui_log_handler)
+                self._gui_log_viewer = GuiLogViewer("dashboard", title="Serena Logs", memory_log_handler=get_memory_log_handler())
+                self._gui_log_viewer.start()
 
         # set the agent context
         if context is None:
@@ -146,8 +152,8 @@ class SerenaAgent:
         tool_names = [tool.get_name_from_cls() for tool in self._all_tools.values()]
 
         # If GUI log window is enabled, set the tool names for highlighting
-        if self._gui_log_handler is not None:
-            self._gui_log_handler.log_viewer.set_tool_names(tool_names)
+        if self._gui_log_viewer is not None:
+            self._gui_log_viewer.set_tool_names(tool_names)
 
         self._tool_usage_stats: ToolUsageStats | None = None
         if self.serena_config.record_tool_usage_stats:
@@ -157,16 +163,17 @@ class SerenaAgent:
 
         # start the dashboard (web frontend), registering its log handler
         if self.serena_config.web_dashboard:
-            dashboard_log_handler = MemoryLogHandler(level=serena_log_level)
-            Logger.root.addHandler(dashboard_log_handler)
             self._dashboard_thread, port = SerenaDashboardAPI(
-                dashboard_log_handler, tool_names, tool_usage_stats=self._tool_usage_stats
+                get_memory_log_handler(), tool_names, tool_usage_stats=self._tool_usage_stats
             ).run_in_thread()
+            dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html"
+            log.info("Serena web dashboard started at %s", dashboard_url)
             if self.serena_config.web_dashboard_open_on_launch:
                 # open the dashboard URL in the default web browser (using a separate process to control
                 # output redirection)
-                process = multiprocessing.Process(target=self._open_dashboard, args=(port,))
+                process = multiprocessing.Process(target=self._open_dashboard, args=(dashboard_url,))
                 process.start()
+                process.join(timeout=1)
 
         # log fundamental information
         log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})")
@@ -253,7 +260,7 @@ class SerenaAgent:
             log.debug(f"Tool usage statistics recording is disabled, not recording usage of '{tool_name}'.")
 
     @staticmethod
-    def _open_dashboard(port: int) -> None:
+    def _open_dashboard(url: str) -> None:
         # Redirect stdout and stderr file descriptors to /dev/null,
         # making sure that nothing can be written to stdout/stderr, even by subprocesses
         null_fd = os.open(os.devnull, os.O_WRONLY)
@@ -262,7 +269,7 @@ class SerenaAgent:
         os.close(null_fd)
 
         # open the dashboard URL in the default web browser
-        webbrowser.open(f"http://localhost:{port}/dashboard/index.html")
+        webbrowser.open(url)
 
     def get_project_root(self) -> str:
         """
@@ -412,10 +419,10 @@ class SerenaAgent:
         """
         project_instance: Project | None = self.serena_config.get_project(project_root_or_name)
         if project_instance is not None:
-            log.info(f"Found registered project {project_instance.project_name} at path {project_instance.project_root}.")
+            log.info(f"Found registered project '{project_instance.project_name}' at path {project_instance.project_root}")
         elif autogenerate and os.path.isdir(project_root_or_name):
             project_instance = self.serena_config.add_project_from_path(project_root_or_name)
-            log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}.")
+            log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}")
         return project_instance
 
     def activate_project_from_path_or_name(self, project_root_or_name: str) -> Project:
@@ -561,7 +568,6 @@ class SerenaAgent:
             assert self.language_server is not None
             self.language_server.save_cache()
             self.language_server.stop()
-        if self._gui_log_handler:
+        if self._gui_log_viewer:
             log.info("Stopping the GUI log window ...")
-            self._gui_log_handler.stop_viewer()
-            Logger.root.removeHandler(self._gui_log_handler)
+            self._gui_log_viewer.stop()

src/serena/cli.py

@@ -2,6 +2,7 @@ import os
 import shutil
 import subprocess
 import sys
+from logging import Logger
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@ from serena.config.serena_config import ProjectConfig, SerenaConfig
 from serena.constants import (
     DEFAULT_CONTEXT,
     DEFAULT_MODES,
+    SERENA_LOG_FORMAT,
     SERENA_MANAGED_DIR_IN_HOME,
     SERENAS_OWN_CONTEXT_YAMLS_DIR,
     SERENAS_OWN_MODE_YAMLS_DIR,
@@ -23,6 +25,7 @@ from serena.constants import (
 )
 from serena.mcp import SerenaMCPFactorySingleProcess
 from serena.project import Project
+from serena.util.logging import MemoryLogHandler
 from solidlsp.ls_config import Language
 
 log = logging.getLogger(__name__)
@@ -135,8 +138,20 @@ class TopLevelCommands(AutoRegisteringGroup):
         trace_lsp_communication: bool | None,
         tool_timeout: float | None,
     ) -> None:
+        # initialize logging, using INFO level initially (will later be adjusted by SerenaAgent according to the config)
+        #   * memory log handler (for use by GUI/Dashboard)
+        #   * stream handler for stderr (for direct console output, which will also be captured by clients like Claude Desktop)
+        # (Note that stdout must never be used for logging, as it is used by the MCP server to communicate with the client.)
+        Logger.root.setLevel(logging.INFO)
+        memory_log_handler = MemoryLogHandler()
+        Logger.root.addHandler(memory_log_handler)
+        stderr_handler = logging.StreamHandler(stream=sys.stderr)
+        stderr_handler.formatter = logging.Formatter(SERENA_LOG_FORMAT)
+        Logger.root.addHandler(stderr_handler)
+
+        log.info("Initializing Serena MCP server")
         project_file = project_file_arg or project
-        factory = SerenaMCPFactorySingleProcess(context=context, project=project_file)
+        factory = SerenaMCPFactorySingleProcess(context=context, project=project_file, memory_log_handler=memory_log_handler)
         server = factory.create_mcp_server(
             host=host,
             port=port,

src/serena/config/serena_config.py

@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar
 import yaml
 from ruamel.yaml.comments import CommentedMap
 from sensai.util import logging
+from sensai.util.logging import LogTime
 from sensai.util.string import ToStringMixin
 
 from serena.constants import (
@@ -163,30 +164,31 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
         project_root = Path(project_root).resolve()
         if not project_root.exists():
             raise FileNotFoundError(f"Project root not found: {project_root}")
-        project_name = project_name or project_root.name
-        if project_language is None:
-            language_composition = determine_programming_language_composition(str(project_root))
-            if len(language_composition) == 0:
-                raise ValueError(
-                    f"No source files found in {project_root}\n\n"
-                    f"To use Serena with this project, you need to either:\n"
-                    f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP)\n"
-                    f"2. Create a project configuration file manually at:\n"
-                    f"   {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n"
-                    f"Example project.yml:\n"
-                    f"  project_name: {project_name}\n"
-                    f"  language: python  # or typescript, java, csharp, rust, go, ruby, cpp, php\n"
-                )
-            # find the language with the highest percentage
-            dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
-        else:
-            dominant_language = project_language.value
-        config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True)
-        config_with_comments["project_name"] = project_name
-        config_with_comments["language"] = dominant_language
-        if save_to_disk:
-            save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True)
-        return cls._from_dict(config_with_comments)
+        with LogTime("Project configuration auto-generation", logger=log):
+            project_name = project_name or project_root.name
+            if project_language is None:
+                language_composition = determine_programming_language_composition(str(project_root))
+                if len(language_composition) == 0:
+                    raise ValueError(
+                        f"No source files found in {project_root}\n\n"
+                        f"To use Serena with this project, you need to either:\n"
+                        f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP)\n"
+                        f"2. Create a project configuration file manually at:\n"
+                        f"   {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n"
+                        f"Example project.yml:\n"
+                        f"  project_name: {project_name}\n"
+                        f"  language: python  # or typescript, java, csharp, rust, go, ruby, cpp, php\n"
+                    )
+                # find the language with the highest percentage
+                dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
+            else:
+                dominant_language = project_language.value
+            config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True)
+            config_with_comments["project_name"] = project_name
+            config_with_comments["language"] = dominant_language
+            if save_to_disk:
+                save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True)
+            return cls._from_dict(config_with_comments)
 
     @classmethod
     def rel_path_to_project_yml(cls) -> str:
@@ -220,7 +222,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
         )
 
     @classmethod
-    def load(cls, project_root: Path | str, autogenerate: bool = True) -> Self:
+    def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self:
         """
         Load a ProjectConfig instance from the path to the project root.
         """
@@ -238,6 +240,54 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
         return cls._from_dict(yaml_data)
 
 
+class RegisteredProject(ToStringMixin):
+    def __init__(self, project_root: str, project_config: "ProjectConfig", project_instance: Optional["Project"] = None) -> None:
+        """
+        Represents a registered project in the Serena configuration.
+
+        :param project_root: the root directory of the project
+        :param project_config: the configuration of the project
+        """
+        self.project_root = Path(project_root).resolve()
+        self.project_config = project_config
+        self._project_instance = project_instance
+
+    def _tostring_exclude_private(self) -> bool:
+        return True
+
+    @property
+    def project_name(self) -> str:
+        return self.project_config.project_name
+
+    @classmethod
+    def from_project_instance(cls, project_instance: "Project") -> "RegisteredProject":
+        return RegisteredProject(
+            project_root=project_instance.project_root,
+            project_config=project_instance.project_config,
+            project_instance=project_instance,
+        )
+
+    def matches_root_path(self, path: str | Path) -> bool:
+        """
+        Check if the given path matches the project root path.
+
+        :param path: the path to check
+        :return: True if the path matches the project root, False otherwise
+        """
+        return self.project_root == Path(path).resolve()
+
+    def get_project_instance(self) -> "Project":
+        """
+        Returns the project instance for this registered project, loading it if necessary.
+        """
+        if self._project_instance is None:
+            from ..project import Project
+
+            with LogTime(f"Loading project instance for {self}", logger=log):
+                self._project_instance = Project(project_root=str(self.project_root), project_config=self.project_config)
+        return self._project_instance
+
+
 @dataclass(kw_only=True)
 class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
     """
@@ -246,7 +296,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
     For testing purposes, it can also be instantiated directly with the desired parameters.
     """
 
-    projects: list["Project"] = field(default_factory=list)
+    projects: list[RegisteredProject] = field(default_factory=list)
     gui_log_window_enabled: bool = False
     log_level: int = logging.INFO
     trace_lsp_communication: bool = False
@@ -317,8 +367,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         """
         Static constructor to create SerenaConfig from the configuration file
         """
-        from ..project import Project
-
         config_file_path = cls._determine_config_file_path()
 
         # create the configuration file from the template if necessary
@@ -355,7 +403,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
                 if path is None:
                     continue
                 num_project_migrations += 1
-            project = Project.load(path)
+            project_config = ProjectConfig.load(path)
+            project = RegisteredProject(
+                project_root=str(path),
+                project_config=project_config,
+            )
             instance.projects.append(project)
 
         # set other configuration parameters
@@ -411,7 +463,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
 
     @cached_property
     def project_paths(self) -> list[str]:
-        return sorted(project.project_root for project in self.projects)
+        return sorted(str(project.project_root) for project in self.projects)
 
     @cached_property
     def project_names(self) -> list[str]:
@@ -424,7 +476,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
             if project.project_config.project_name == project_root_or_name:
                 project_candidates.append(project)
         if len(project_candidates) == 1:
-            return project_candidates[0]
+            return project_candidates[0].get_project_instance()
         elif len(project_candidates) > 1:
             raise ValueError(
                 f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. "
@@ -432,10 +484,9 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
             )
         # no project found by name; check if it's a path
         if os.path.isdir(project_root_or_name):
-            project_root = Path(project_root_or_name).resolve()
             for project in self.projects:
-                if Path(project.project_root).resolve() == project_root:
-                    return project
+                if project.matches_root_path(project_root_or_name):
+                    return project.get_project_instance()
         return None
 
     def add_project_from_path(self, project_root: Path | str) -> "Project":
@@ -463,14 +514,14 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         project_config = ProjectConfig.load(project_root, autogenerate=True)
 
         new_project = Project(project_root=str(project_root), project_config=project_config, is_newly_created=True)
-        self.projects.append(new_project)
+        self.projects.append(RegisteredProject.from_project_instance(new_project))
         self.save()
 
         return new_project
 
     def remove_project(self, project_name: str) -> None:
         # find the index of the project with the desired name and remove it
-        for i, project in enumerate(self.projects):
+        for i, project in enumerate(list(self.projects)):
             if project.project_name == project_name:
                 del self.projects[i]
                 break
@@ -488,5 +539,5 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         loaded_original_yaml = deepcopy(self.loaded_commented_yaml)
         # projects are unique absolute paths
         # we also canonicalize them before saving
-        loaded_original_yaml["projects"] = sorted({str(Path(project.project_root).resolve()) for project in self.projects})
+        loaded_original_yaml["projects"] = sorted({str(project.project_root) for project in self.projects})
         save_yaml(self.config_file_path, loaded_original_yaml, preserve_comments=True)

src/serena/dashboard.py

@@ -1,5 +1,4 @@
 import os
-import queue
 import socket
 import threading
 from collections.abc import Callable
@@ -10,7 +9,8 @@ from pydantic import BaseModel
 from sensai.util import logging
 
 from serena.analytics import ToolUsageStats
-from serena.constants import SERENA_DASHBOARD_DIR, SERENA_LOG_FORMAT
+from serena.constants import SERENA_DASHBOARD_DIR
+from serena.util.logging import MemoryLogHandler
 
 log = logging.getLogger(__name__)
 
@@ -18,43 +18,6 @@ log = logging.getLogger(__name__)
 logging.getLogger("werkzeug").setLevel(logging.WARNING)
 
 
-class MemoryLogHandler(logging.Handler):
-    def __init__(self, level: int = logging.NOTSET) -> None:
-        super().__init__(level=level)
-        self.setFormatter(logging.Formatter(SERENA_LOG_FORMAT))
-        self._log_buffer = LogBuffer()
-        self._log_queue: queue.Queue[str] = queue.Queue()
-        self._stop_event = threading.Event()
-
-        # start background thread to process logs
-        self.worker_thread = threading.Thread(target=self._process_queue, daemon=True)
-        self.worker_thread.start()
-
-    def emit(self, record: logging.LogRecord) -> None:
-        msg = self.format(record)
-        self._log_queue.put_nowait(msg)
-
-    def _process_queue(self) -> None:
-        while not self._stop_event.is_set():
-            try:
-                msg = self._log_queue.get(timeout=1)
-                self._log_buffer.append(msg)
-                self._log_queue.task_done()
-            except queue.Empty:
-                continue
-
-    def get_log_messages(self) -> list[str]:
-        return self._log_buffer.logs
-
-
-class LogBuffer:
-    def __init__(self) -> None:
-        self.logs: list[str] = []
-
-    def append(self, msg: str) -> None:
-        self.logs.append(msg)
-
-
 class RequestLog(BaseModel):
     start_idx: int = 0
 

src/serena/gui_log_viewer.py

@@ -4,7 +4,6 @@ import os
 import queue
 import sys
 import threading
-import time
 import tkinter as tk
 import traceback
 from enum import Enum, auto
@@ -12,6 +11,7 @@ from pathlib import Path
 from typing import Literal
 
 from serena import constants
+from serena.util.logging import MemoryLogHandler
 
 log = logging.getLogger(__name__)
 
@@ -31,11 +31,20 @@ class GuiLogViewer:
     It can also highlight tool names in boldface when they appear in log messages.
     """
 
-    def __init__(self, mode: Literal["dashboard", "error"], title="Log Viewer", width=800, height=600):
+    def __init__(
+        self,
+        mode: Literal["dashboard", "error"],
+        title="Log Viewer",
+        memory_log_handler: MemoryLogHandler | None = None,
+        width=800,
+        height=600,
+    ):
         """
         :param mode: the mode; if "dashboard", run a dashboard with logs and some control options; if "error", run
             a simple error log viewer (for fatal exceptions)
         :param title: the window title
+        :param memory_log_handler: an optional log handler from which to obtain log messages; If not provided,
+            must pass the instance to a `GuiLogViewerHandler` to add log messages.
         :param width: the initial window width
         :param height: the initial window height
         """
@@ -57,13 +66,14 @@ class GuiLogViewer:
             LogLevel.DEFAULT: "#000000",  # Black
         }
 
-    def print_status(self, s):
-        print(s + "\n", file=sys.stderr)
+        if memory_log_handler is not None:
+            for msg in memory_log_handler.get_log_messages():
+                self.message_queue.put(msg)
+            memory_log_handler.add_emit_callback(lambda msg: self.message_queue.put(msg))
 
     def start(self):
         """Start the log viewer in a separate thread."""
         if not self.running:
-            self.print_status("Starting thread")
             self.log_thread = threading.Thread(target=self.run_gui)
             self.log_thread.daemon = True
             self.log_thread.start()
@@ -381,23 +391,15 @@ class GuiLogViewerHandler(logging.Handler):
             self.log_viewer.stop()
 
 
-def show_fatal_exception(e: Exception, duration_secs: int = 60):
+def show_fatal_exception(e: Exception):
     """
     Makes sure the given exception is shown in the GUI log viewer,
     either an existing instance or a new one.
 
     :param e: the exception to display
-    :param duration_secs: the duration for which to display the error before
-        terminating the program for the case where the log viewer is already present
-        in a daemon thread which will terminate when the program does
     """
-    if GuiLogViewerHandler.is_instance_registered():
-        # show in existing daemon thread, waiting for the given duration
-        log.error(f"Fatal error: {e}", exc_info=e)
-        time.sleep(duration_secs)
-    else:
-        # show in new window in main thread (user must close it)
-        log_viewer = GuiLogViewer("error")
-        exc_info = "".join(traceback.format_exception(type(e), e, e.__traceback__))
-        log_viewer.add_log(f"ERROR Fatal exception: {e}\n{exc_info}")
-        log_viewer.run_gui()
+    # show in new window in main thread (user must close it)
+    log_viewer = GuiLogViewer("error")
+    exc_info = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+    log_viewer.add_log(f"ERROR Fatal exception: {e}\n{exc_info}")
+    log_viewer.run_gui()

src/serena/mcp.py

@@ -7,7 +7,6 @@ from abc import abstractmethod
 from collections.abc import AsyncIterator, Iterator, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
-from logging import Formatter, Logger, StreamHandler
 from typing import Any, Literal, cast
 
 import docstring_parser
@@ -22,21 +21,19 @@ from serena.agent import (
     SerenaConfig,
 )
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES
+from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES, SERENA_LOG_FORMAT
 from serena.tools import Tool
 from serena.util.exception import show_fatal_exception_safe
+from serena.util.logging import MemoryLogHandler
 
 log = logging.getLogger(__name__)
-LOG_FORMAT = "%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s"
-LOG_LEVEL = logging.INFO
 
 
 def configure_logging(*args, **kwargs) -> None:  # type: ignore
-    # configure logging to stderr (will be captured by Claude Desktop); stdio is the MCP communication stream and cannot be used!
-    Logger.root.setLevel(LOG_LEVEL)
-    handler = StreamHandler(stream=sys.stderr)
-    handler.formatter = Formatter(LOG_FORMAT)
-    Logger.root.addHandler(handler)
+    # We only do something here if logging has not yet been configured.
+    # Normally, logging is configured in the MCP server startup script.
+    if not logging.is_enabled():
+        logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=SERENA_LOG_FORMAT)
 
 
 # patch the logging configuration function in fastmcp, because it's hard-coded and broken
@@ -190,7 +187,7 @@ class SerenaMCPFactorySingleProcess(SerenaMCPFactory):
     MCP server factory where the SerenaAgent and its language server run in the same process as the MCP server
     """
 
-    def __init__(self, context: str = DEFAULT_CONTEXT, project: str | None = None):
+    def __init__(self, context: str = DEFAULT_CONTEXT, project: str | None = None, memory_log_handler: MemoryLogHandler | None = None):
         """
         :param context: The context name or path to context file
         :param project: Either an absolute path to the project directory or a name of an already registered project.
@@ -199,9 +196,12 @@ class SerenaMCPFactorySingleProcess(SerenaMCPFactory):
         """
         super().__init__(context=context, project=project)
         self.agent: SerenaAgent | None = None
+        self.memory_log_handler = memory_log_handler
 
     def _instantiate_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> None:
-        self.agent = SerenaAgent(project=self.project, serena_config=serena_config, context=self.context, modes=modes)
+        self.agent = SerenaAgent(
+            project=self.project, serena_config=serena_config, context=self.context, modes=modes, memory_log_handler=self.memory_log_handler
+        )
 
     def _iter_tools(self) -> Iterator[Tool]:
         assert self.agent is not None

src/serena/util/logging.py

@@ -0,0 +1,67 @@
+import queue
+import threading
+from collections.abc import Callable
+
+from sensai.util import logging
+
+from serena.constants import SERENA_LOG_FORMAT
+
+
+class MemoryLogHandler(logging.Handler):
+    def __init__(self, level: int = logging.NOTSET) -> None:
+        super().__init__(level=level)
+        self.setFormatter(logging.Formatter(SERENA_LOG_FORMAT))
+        self._log_buffer = LogBuffer()
+        self._log_queue: queue.Queue[str] = queue.Queue()
+        self._stop_event = threading.Event()
+        self._emit_callbacks: list[Callable[[str], None]] = []
+
+        # start background thread to process logs
+        self.worker_thread = threading.Thread(target=self._process_queue, daemon=True)
+        self.worker_thread.start()
+
+    def add_emit_callback(self, callback: Callable[[str], None]) -> None:
+        """
+        Adds a callback that will be called with each log message.
+        The callback should accept a single string argument (the log message).
+        """
+        self._emit_callbacks.append(callback)
+
+    def emit(self, record: logging.LogRecord) -> None:
+        msg = self.format(record)
+        self._log_queue.put_nowait(msg)
+
+    def _process_queue(self) -> None:
+        while not self._stop_event.is_set():
+            try:
+                msg = self._log_queue.get(timeout=1)
+                self._log_buffer.append(msg)
+                for callback in self._emit_callbacks:
+                    try:
+                        callback(msg)
+                    except:
+                        pass
+                self._log_queue.task_done()
+            except queue.Empty:
+                continue
+
+    def get_log_messages(self) -> list[str]:
+        return self._log_buffer.get_log_messages()
+
+
+class LogBuffer:
+    """
+    A thread-safe buffer for storing log messages.
+    """
+
+    def __init__(self) -> None:
+        self._log_messages: list[str] = []
+        self._lock = threading.Lock()
+
+    def append(self, msg: str) -> None:
+        with self._lock:
+            self._log_messages.append(msg)
+
+    def get_log_messages(self) -> list[str]:
+        with self._lock:
+            return self._log_messages.copy()

test/serena/test_serena_agent.py

@@ -7,7 +7,7 @@ import pytest
 
 import test.solidlsp.clojure as clj
 from serena.agent import SerenaAgent
-from serena.config.serena_config import ProjectConfig, SerenaConfig
+from serena.config.serena_config import ProjectConfig, RegisteredProject, SerenaConfig
 from serena.project import Project
 from serena.tools import FindReferencingSymbolsTool, FindSymbolTool
 from solidlsp.ls_config import Language
@@ -45,7 +45,7 @@ def serena_config():
                     encoding="utf-8",
                 ),
             )
-            test_projects.append(project)
+            test_projects.append(RegisteredProject.from_project_instance(project))
 
     config = SerenaConfig(gui_log_window_enabled=False, web_dashboard=False, log_level=logging.ERROR)
     config.projects = test_projects