From 8b946790490724ea5c6c5c08f9b5cc778c87e25a Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sat, 5 Apr 2025 17:37:07 +0200 Subject: [PATCH 01/11] Don't rely on server_lifespan to start the server --- src/multilspy/language_server.py | 8 ++++- src/serena/agent.py | 60 +++++++++++++++++++++----------- src/serena/mcp.py | 34 ++++++++++++------ 3 files changed, 69 insertions(+), 33 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 41fc43c..3ae9780 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -1827,7 +1827,13 @@ class SyncLanguageServer: self._server_context = self.language_server.start_server() asyncio.run_coroutine_threadsafe(self._server_context.__aenter__(), loop=self.loop).result() return self - + + def is_running(self) -> bool: + """ + Check if the language server is running. + """ + return self.loop is not None and self.loop_thread is not None and self.loop_thread.is_alive() + def stop(self) -> None: """ Shuts down the language server process and cleans up resources. diff --git a/src/serena/agent.py b/src/serena/agent.py index b98efaa..c428c19 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -10,8 +10,7 @@ import sys import traceback from abc import ABC from collections import defaultdict -from collections.abc import Callable, Generator, Iterable, Iterator -from contextlib import contextmanager +from collections.abc import Callable, Generator, Iterable from logging import Logger from pathlib import Path from typing import Any, TypeVar, cast @@ -93,9 +92,9 @@ class SerenaAgent: ) # create and start the language server instance - config = MultilspyConfig(code_language=self.language) - logger = MultilspyLogger() - self.language_server = SyncLanguageServer.create(config, logger, self.project_root) + self._config = MultilspyConfig(code_language=self.language) + self._ls_logger = MultilspyLogger() + self.language_server = SyncLanguageServer.create(self._config, self._ls_logger, self.project_root) self.prompt_factory = PromptFactory() self.symbol_manager = SymbolManager(self.language_server, self) @@ -123,6 +122,14 @@ class SerenaAgent: if self._start_language_server: log.info("Starting the language server ...") self.language_server.start() + if not self.language_server.is_running(): + raise RuntimeError(f"Failed to start the language server for {self.language} and project {self.project_root}") + + def reset_language_server(self) -> None: + self.language_server.stop() + self.language_server.start() + if not self.language_server.is_running(): + raise RuntimeError(f"Failed to start (reset) the language server for {self.language} and project {self.project_root}") def get_tool(self, tool_class: type[TTool]) -> TTool: return self._all_tools[tool_class] # type: ignore @@ -151,16 +158,6 @@ class SerenaAgent: self._gui_log_handler.stop_viewer() Logger.root.removeHandler(self._gui_log_handler) - @contextmanager - def language_server_lifecycle_context(self) -> Iterator[None]: - """ - Context manager for the language server's lifecycle - """ - if self._start_language_server: - raise Exception("This context manager can only be used if the instance is created with start_language_server=True") - with self.language_server.start_server(): - yield - class MemoriesManager: def __init__(self, memory_dir: str): @@ -195,12 +192,30 @@ class MemoriesManager: class Component(ABC): def __init__(self, agent: "SerenaAgent"): self.agent = agent - self.language_server = agent.language_server - self.project_root = agent.project_root - self.project_config = agent.project_config - self.prompt_factory = agent.prompt_factory - self.memories_manager = agent.memories_manager - self.symbol_manager = agent.symbol_manager + + @property + def language_server(self) -> SyncLanguageServer: + return self.agent.language_server + + @property + def project_root(self) -> str: + return self.agent.project_root + + @property + def project_config(self) -> dict[str, Any]: + return self.agent.project_config + + @property + def prompt_factory(self) -> PromptFactory: + return self.agent.prompt_factory + + @property + def memories_manager(self) -> MemoriesManager: + return self.agent.memories_manager + + @property + def symbol_manager(self) -> SymbolManager: + return self.agent.symbol_manager _DEFAULT_MAX_ANSWER_LENGTH = int(2e5) @@ -261,6 +276,9 @@ class Tool(Component): """ Applies the tool with the given arguments """ + if not self.language_server.is_running(): + self.agent.reset_language_server() + apply_fn = getattr(self, "apply") if apply_fn is None: raise ValueError(f"apply not defined in {self}") diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 7d815f1..871aa96 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -13,7 +13,6 @@ from mcp.server.fastmcp.server import FastMCP, Settings from mcp.server.fastmcp.tools.base import Tool as MCPTool from mcp.server.fastmcp.utilities.func_metadata import func_metadata from sensai.util import logging -from sensai.util.helper import mark_used from serena.agent import SerenaAgent, Tool @@ -43,8 +42,6 @@ def make_tool( tool: Tool, ) -> MCPTool: """Create a Tool from a function.""" - from mcp.server.fastmcp import Context - func_name = tool.get_name() apply_fn = getattr(tool, "apply") @@ -57,8 +54,7 @@ def make_tool( func_arg_metadata = func_metadata(apply_fn) parameters = func_arg_metadata.arg_model.model_json_schema() - def execute_fn(ctx: Context, **kwargs) -> str: # type: ignore - mark_used(ctx) + def execute_fn(**kwargs) -> str: # type: ignore return tool.apply_ex(log_call=True, catch_exceptions=True, **kwargs) return MCPTool( @@ -68,10 +64,23 @@ def make_tool( parameters=parameters, fn_metadata=func_arg_metadata, is_async=is_async, - context_kwarg="ctx", + context_kwarg=None, ) +_SERENA_AGENT_REGISTRY: dict[str, SerenaAgent] = {} +"""Map of project file paths to their corresponding SerenaAgent instances.""" + + +def _get_create_serena_agent(project_file_path: str) -> SerenaAgent: + if project_file_path not in _SERENA_AGENT_REGISTRY: + log.info(f"Creating new SerenaAgent for project file: {project_file_path}") + _SERENA_AGENT_REGISTRY[project_file_path] = SerenaAgent(project_file_path, start_language_server=True) + else: + log.info(f"Reusing existing SerenaAgent for project file: {project_file_path}") + return _SERENA_AGENT_REGISTRY[project_file_path] + + def create_mcp_server() -> FastMCP: argv = sys.argv[1:] if len(argv) != 1: @@ -79,19 +88,22 @@ def create_mcp_server() -> FastMCP: sys.exit(1) project_file_path = argv[0] - agent = SerenaAgent(project_file_path) + + agent = _get_create_serena_agent(project_file_path) @asynccontextmanager - async def server_lifespan(mcp_server: FastMCP) -> AsyncIterator[SerenaMCPRequestContext]: + async def server_lifespan(mcp_server: FastMCP) -> AsyncIterator[None]: """Manage server startup and shutdown lifecycle.""" - with agent.language_server_lifecycle_context(): - yield SerenaMCPRequestContext(agent=agent) + try: + yield + finally: + for agent in _SERENA_AGENT_REGISTRY.values(): + agent.language_server.stop() mcp_settings = Settings(lifespan=server_lifespan) mcp = FastMCP(**mcp_settings.model_dump()) for tool in agent.tools.values(): mcp._tool_manager._tools[tool.get_name()] = make_tool(tool) - return mcp From a10fc55485e133ead256b29a800ae2811c693e8c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sat, 5 Apr 2025 18:00:43 +0200 Subject: [PATCH 02/11] Reinstated saving LS cache --- src/multilspy/language_server.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 3ae9780..61d2072 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -1397,8 +1397,14 @@ class LanguageServer: if self._cache_has_changed: self.logger.log(f"Saving updated document symbols cache to {self._cache_path}", logging.INFO) self._cache_path.parent.mkdir(parents=True, exist_ok=True) - with open(self._cache_path, "wb") as f: - pickle.dump(self._document_symbols_cache, f) + try: + with open(self._cache_path, "wb") as f: + pickle.dump(self._document_symbols_cache, f) + except Exception as e: + self.logger.log( + f"Failed to save document symbols cache to {self._cache_path}: {e}. " + "Note: this may have resulted in a corrupted cache file.", logging.ERROR + ) def load_cache(self): if not self._cache_path.exists(): @@ -1407,9 +1413,13 @@ class LanguageServer: with open(self._cache_path, "rb") as f: try: self._document_symbols_cache = pickle.load(f) - except: + except Exception as e: # cache often becomes corrupt, so just skip loading it - pass + self.logger.log( + f"Failed to load document symbols cache from {self._cache_path}: {e}. Possible cause: the cache file is corrupted. " + "Check for any errors related to saving the cache in the logs.", + logging.ERROR + ) @ensure_all_methods_implemented(LanguageServer) @@ -1847,6 +1857,7 @@ class SyncLanguageServer: self.loop_thread.join() self.loop = None self.loop_thread = None + self.save_cache() def save_cache(self): """ From b19a564c9fe002babf678f8018cb936f4080f38c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sun, 6 Apr 2025 13:32:25 +0200 Subject: [PATCH 03/11] LS: stop no longer raises if LS is not running --- src/multilspy/language_server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 61d2072..6f22540 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -1847,11 +1847,14 @@ class SyncLanguageServer: def stop(self) -> None: """ Shuts down the language server process and cleans up resources. - Must be called after start(). + + If the language server is not running, this method will log a warning and do nothing. """ - if not self.loop or not self.loop_thread: - raise MultilspyException("Language Server not started") - + if not self.is_running(): + self.language_server.logger.log("Language server not running, skipping shutdown.", logging.INFO) + return + + assert self.loop asyncio.run_coroutine_threadsafe(self._server_context.__aexit__(None, None, None), loop=self.loop).result() self.loop.call_soon_threadsafe(self.loop.stop) self.loop_thread.join() From 4c92934c8a53b0ebd6e20c5142157f7a0f536bf1 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Apr 2025 18:04:41 +0200 Subject: [PATCH 04/11] Fix agno logging configuration being broken by reimport (disabling GUI logger) --- scripts/agno_agent.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/agno_agent.py b/scripts/agno_agent.py index 47f4fed..de2c771 100644 --- a/scripts/agno_agent.py +++ b/scripts/agno_agent.py @@ -9,7 +9,10 @@ from serena.agno import SerenaAgnoAgentProvider mark_used(Gemini, Claude) -logging.configure(level=logging.INFO) +# initialize logging (Note: since this module is reimported by serve_playground_app and the logging configuration +# is extended by SerenaAgentProvider, we must handle this here conditionally) +if __name__ == "__main__": + logging.configure(level=logging.INFO) # Define the model to use (see Agno documentation for supported models; these are just examples) model = Claude(id="claude-3-7-sonnet-20250219") @@ -18,4 +21,4 @@ model = Claude(id="claude-3-7-sonnet-20250219") app = Playground(agents=[SerenaAgnoAgentProvider.get_agent(model)]).get_app() if __name__ == "__main__": - serve_playground_app("agno_agent:app", reload=False) + serve_playground_app("agno_agent:app", reload=False, log_config=None) From 2e1e5e59aaf8b2f5eb5873bd7bab4e911fea642f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Apr 2025 19:49:08 +0200 Subject: [PATCH 05/11] Support project changes from within Serena (via tools) * Establish new Serena-wide configuration file serena_config.yml with * Setting whether to enable project activation * List of project configuration files * GUI logger configuration (no longer part of project configuration) * Add new tools for project activation (activate_project, get_active_project) * Project configuration file is no longer required for startup (now optional): If project activation is enabled, no project must be provided (but at least one project must be configured) --- README.md | 76 +++-- myproject.demo.yml => myproject.template.yml | 98 +++--- serena_config.template.yml | 28 ++ src/serena/__init__.py | 2 +- src/serena/agent.py | 320 +++++++++++++++---- src/serena/agno.py | 25 +- src/serena/gui_log_viewer.py | 7 + src/serena/mcp.py | 67 ++-- 8 files changed, 434 insertions(+), 189 deletions(-) rename myproject.demo.yml => myproject.template.yml (55%) create mode 100644 serena_config.template.yml diff --git a/README.md b/README.md index 7564734..147c664 100644 --- a/README.md +++ b/README.md @@ -128,12 +128,23 @@ Serena can read, write and execute code, read logs and the terminal output. ## Quick Start -### MCP Server (Claude Desktop) +### Setup and Configuration 1. Install `uv` (instructions [here](https://docs.astral.sh/uv/getting-started/installation/)) 2. Clone the repository to `/path/to/serena`. -3. Create a configuration file for your project, say `myproject.yml` based on the template in [myproject.demo.yml](myproject.demo.yml). -4. Configure the MCP server in your client. +3. Copy `serena_config.template.yml` to `serena_config.yml` and adjust settings. +4. Copy `myproject.template.yml` to `myproject.yml` and adjust the settings specific to your project. + (Add one such file for each project you want Serena to work on.) +5. If you want Serena to dynamically switch between projects, add the list of all project files + created in the previous step to the `projects` list in `serena_config.yml`. + +After this initial setup, continue with one of the sections below, depending on how you +want to use Serena. + +### MCP Server (Claude Desktop) + +1. Create a configuration file for your project, say `myproject.yml` based on the template in [myproject.template.yml](myproject.template.yml). +2. Configure the MCP server in your client. For [Claude Desktop](https://claude.ai/download) (available for Windows and macOS), go to File / Settings / Developer / MCP Servers / Edit Config, which will let you open the JSON file `claude_desktop_config.json`. Add the following (with adjusted paths) to enable Serena: @@ -147,6 +158,9 @@ Serena can read, write and execute code, read logs and the terminal output. } } ``` + + :info: The path to the project file is optional if you have set `enable_project_activation` in your configuration, + as this setting will allow you to simply instruct Claude to activate the project you want to work on. If you are using paths containing backslashes for paths on Windows (note that you can also just use forward slashes), be sure to escape them correctly (`\\`). @@ -549,31 +563,33 @@ For details on contributing, see [here](/CONTRIBUTING.md). ## Full List of Tools -Here the full list of Serena's default tools with a short description (the output of `uv run serena-list-tools`) - - * `check_onboarding_performed`: Checks whether the onboarding was already performed. - * `create_text_file`: Creates/overwrites a file in the project directory. - * `delete_lines`: Deletes a range of lines within a file. - * `delete_memory`: Deletes a memory from Serena's project-specific memory store. - * `execute_shell_command`: Executes a shell command. - * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). - * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). - * `get_dir_overview`: Gets an overview of the top-level symbols defined in all files within a given directory. - * `get_document_overview`: Gets an overview of the top-level symbols defined in a given file. - * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. - * `insert_at_line`: Inserts content at a given line in a file. - * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. - * `list_dir`: Lists files and directories in the given directory (optionally with recursion). - * `list_memories`: Lists memories in Serena's project-specific memory store. - * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). - * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). - * `read_file`: Reads a file within the project directory. - * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. - * `replace_symbol_body`: Replaces the full definition of a symbol. - * `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project. - * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. - * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. - * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. - * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. - * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. +Here is the full list of Serena's tools with a short description (output of `uv run serena-list-tools`): +* `activate_project`: Activates a project by name. +* `check_onboarding_performed`: Checks whether the onboarding was already performed. +* `create_text_file`: Creates/overwrites a file in the project directory. +* `delete_lines`: Deletes a range of lines within a file. +* `delete_memory`: Deletes a memory from Serena's project-specific memory store. +* `execute_shell_command`: Executes a shell command. +* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). +* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). +* `get_active_project`: Gets the name of the currently active project (if any) and lists existing projects +* `get_dir_overview`: Gets an overview of the top-level symbols defined in all files within a given directory. +* `get_document_overview`: Gets an overview of the top-level symbols defined in a given file. +* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +* `insert_at_line`: Inserts content at a given line in a file. +* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +* `list_dir`: Lists files and directories in the given directory (optionally with recursion). +* `list_memories`: Lists memories in Serena's project-specific memory store. +* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). +* `read_file`: Reads a file within the project directory. +* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +* `replace_lines`: Replaces a range of lines within a file with new content. +* `replace_symbol_body`: Replaces the full definition of a symbol. +* `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project. +* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. +* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. \ No newline at end of file diff --git a/myproject.demo.yml b/myproject.template.yml similarity index 55% rename from myproject.demo.yml rename to myproject.template.yml index 74c277d..d1c4cce 100644 --- a/myproject.demo.yml +++ b/myproject.template.yml @@ -1,56 +1,42 @@ -# absolute path to the project -project_root: /path/to/project - -# language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby) -# Special requirements: -# * csharp: Requires the presence of a .sln file in the project folder. -language: python - -# list of directories to ignore: either names (e.g. "temp") or relative paths (e.g. "build/foo") -ignored_dirs: [".git", "temp"] - -# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. -# Below is the complete list of tools for convenience. -# To make sure you have the latest list of tools, and to view their descriptions, -# execute `uv run serena-list-tools`. -# -# check_onboarding_performed -# create_text_file -# delete_lines -# delete_memory -# execute_shell_command -# find_referencing_symbols -# find_symbol -# get_dir_overview -# get_document_overview -# insert_after_symbol -# insert_at_line -# insert_before_symbol -# list_dir -# list_memories -# onboarding -# prepare_for_new_conversation -# read_file -# read_memory -# replace_symbol_body -# search_in_all_code -# summarize_changes -# think_about_collected_information -# think_about_task_adherence -# think_about_whether_you_are_done -# write_memory -excluded_tools: [] - -# Whether to open a graphical window with Serena's logs (not supported on MacOS). -# This is useful both for troubleshooting and for monitoring the tool calls, -# especially when using the agno playground, since the tool calls are not always shown, -# and the input params are never shown in the agno UI. -# When used as MCP server for Claude Desktop, the logs are primarily for troubleshooting. -# Note: unfortunately, the various entities starting the Serena server or agent do so in -# mysterious ways, often starting multiple instances of the process without shutting down -# previous instances. This leads to multiple log windows being opened, and only the last -# window being updated. Since we can't control how agno or Claude Desktop starts Serena, we have to live with this limitation for now. -gui_log_window: True - -# minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error) -gui_log_level: 20 +# absolute path to the project +project_root: /path/to/project + +# language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby) +# Special requirements: +# * csharp: Requires the presence of a .sln file in the project folder. +language: python + +# list of directories to ignore: either names (e.g. "temp") or relative paths (e.g. "build/foo") +ignored_dirs: [".git", "temp"] + +# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details. +# Below is the complete list of tools for convenience. +# To make sure you have the latest list of tools, and to view their descriptions, +# execute `uv run serena-list-tools`. +# +# check_onboarding_performed +# create_text_file +# delete_lines +# delete_memory +# execute_shell_command +# find_referencing_symbols +# find_symbol +# get_dir_overview +# get_document_overview +# insert_after_symbol +# insert_at_line +# insert_before_symbol +# list_dir +# list_memories +# onboarding +# prepare_for_new_conversation +# read_file +# read_memory +# replace_symbol_body +# search_in_all_code +# summarize_changes +# think_about_collected_information +# think_about_task_adherence +# think_about_whether_you_are_done +# write_memory +excluded_tools: [] diff --git a/serena_config.template.yml b/serena_config.template.yml new file mode 100644 index 0000000..989f2e3 --- /dev/null +++ b/serena_config.template.yml @@ -0,0 +1,28 @@ +# whether to enable project activation/switching between the projects defined below using the +# "activate_project" command. If this is enabled, you can instruct the agent to switch to a project +# by its name (see "projects" below). +# Note that if this is enabled, then per-project tool activation will be handled within Serena, +# i.e. Serena will provide all tools but will disallow the execution of tools that are not enabled for +# the current project. This is because clients typically do not respond to a changed set of tools. +enable_project_activation: True + +# Add your list of .yml project files here (which you can switch between using "activate_project"). +# Serena will know the projects by their base filename, i.e. "MyProject.yml" will be project "MyProject" to Serena. +# Paths should either be relative to the directory this configuration file is in or absolute. +projects: + - myproject.yml + +# Whether to open a graphical window with Serena's logs (not supported on macOS). +# This is useful both for troubleshooting and for monitoring the tool calls, +# especially when using the agno playground, since the tool calls are not always shown, +# and the input params are never shown in the agno UI. +# When used as MCP server for Claude Desktop, the logs are primarily for troubleshooting. +# Note: unfortunately, the various entities starting the Serena server or agent do so in +# mysterious ways, often starting multiple instances of the process without shutting down +# previous instances. This leads to multiple log windows being opened, and only the last +# window being updated. Since we can't control how agno or Claude Desktop start Serena, +# we have to live with this limitation for now. +gui_log_window: True + +# minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error) +gui_log_level: 20 diff --git a/src/serena/__init__.py b/src/serena/__init__.py index aa295c0..9a6ae1b 100644 --- a/src/serena/__init__.py +++ b/src/serena/__init__.py @@ -4,4 +4,4 @@ __version__ = "0.1.0-dev1" def serena_root_path() -> str: from pathlib import Path - return str(Path(__file__).parent.parent.absolute()) + return str(Path(__file__).parent.parent.parent.absolute()) diff --git a/src/serena/agent.py b/src/serena/agent.py index b598213..cc7b4fe 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,26 +6,27 @@ import inspect import json import os import platform -import sys import traceback from abc import ABC from collections import defaultdict from collections.abc import Callable, Generator, Iterable from logging import Logger from pathlib import Path -from typing import Any, TypeVar, cast +from typing import Any, Self, TypeVar, cast import yaml from sensai.util import logging -from sensai.util.string import dict_string +from sensai.util.string import ToStringMixin, dict_string from multilspy import SyncLanguageServer from multilspy.multilspy_config import Language, MultilspyConfig from multilspy.multilspy_logger import MultilspyLogger from multilspy.multilspy_types import SymbolKind +from serena import serena_root_path from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler from serena.llm.prompt_factory import PromptFactory from serena.symbol import SymbolLocation, SymbolManager +from serena.util.class_decorators import singleton from serena.util.file_system import scan_directory from serena.util.inspection import iter_subclasses from serena.util.shell import execute_shell_command @@ -36,6 +37,67 @@ TTool = TypeVar("TTool", bound="Tool") SUCCESS_RESULT = "OK" +class ProjectConfig(ToStringMixin): + SERENA_MANAGED_DIR = ".serena" + + def __init__(self, config_dict: dict[str, Any], project_name: str): + self.project_name: str = project_name + self.language: Language = Language(config_dict["language"]) + self.project_root: str = str(Path(config_dict["project_root"]).resolve()) + self.ignored_dirs: list[str] = config_dict.get("ignored_dirs", []) + self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) + + def _tostring_excludes(self) -> list[str]: + return ["ignored_dirs", "excluded_tools"] + + @classmethod + def project_name_from_yml_path(cls, yml_file_path: str | Path) -> str: + return Path(yml_file_path).stem + + @classmethod + def from_yml(cls, yml_path: str | Path) -> Self: + with open(yml_path, encoding="utf-8") as f: + config_dict = yaml.safe_load(f) + return cls(config_dict, cls.project_name_from_yml_path(yml_path)) + + def get_serena_managed_dir(self) -> str: + return os.path.join(self.project_root, self.SERENA_MANAGED_DIR) + + +@singleton +class SerenaConfig: + """ + Handles user-defined Serena configuration based on the configuration file + """ + + CONFIG_FILE = "serena_config.yml" + + def __init__(self) -> None: + config_file = os.path.join(serena_root_path(), self.CONFIG_FILE) + if not os.path.exists(config_file): + raise FileNotFoundError(f"Serena configuration file not found: {config_file}") + with open(config_file, encoding="utf-8") as f: + config_yaml = yaml.safe_load(f) + + self.projects: dict[str, Path] = {} + for project_yaml_path in config_yaml["projects"]: + project_yaml_path = Path(project_yaml_path) + if not project_yaml_path.is_absolute(): + project_yaml_path = Path(serena_root_path()) / project_yaml_path + project_name = project_yaml_path.stem + self.projects[project_name] = project_yaml_path + self.project_names = list(self.projects.keys()) + + self.gui_log_window_enabled = config_yaml.get("gui_log_window", True) + self.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) + self.enable_project_activation = config_yaml.get("enable_project_activation", True) + + def read_project_configuration(self, project_name: str) -> ProjectConfig: + if project_name not in self.projects: + raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}") + return ProjectConfig.from_yml(self.projects[project_name]) + + class LinesRead: def __init__(self) -> None: self.files: dict[str, set[tuple[int, int]]] = defaultdict(lambda: set()) @@ -53,94 +115,149 @@ class LinesRead: class SerenaAgent: - def __init__(self, project_file_path: str, start_language_server: bool = False): + def __init__(self, project_file_path: str | None = None, project_activation_callback: Callable[[], None] | None = None): """ - :param project_file_path: the project configuration file path (.yml) - :param start_language_server: whether to start the language server immediately and manage its - lifecycle internally + :param project_file_path: the configuration file (.yml) of the project to load immediately; + if None, do not load any project (must use project selection tool to activate a project). + If a project is provided, the corresponding language server will be started. + :param project_activation_callback: a callback function to be called when a project is activated. """ - self._start_language_server = start_language_server + # obtain serena configuration + self.serena_config = SerenaConfig() - if not os.path.exists(project_file_path): - print(f"Project file not found: {project_file_path}", file=sys.stderr) - sys.exit(1) - - # read project configuration - with open(project_file_path, encoding="utf-8") as f: - project_config = yaml.safe_load(f) - self.project_config = project_config - self.language = Language(project_config["language"]) - self.project_root = str(Path(project_config["project_root"]).resolve()) - - # enable GUI log window - enable_gui_log = project_config.get("gui_log_window", True) - self._gui_log_handler = None - if enable_gui_log: + # open GUI log window if enabled + self._gui_log_handler: GuiLogViewerHandler | None = None + if self.serena_config.gui_log_window_enabled: if platform.system() == "Darwin": log.warning("GUI log window is not supported on macOS") else: - log_level = project_config.get("gui_log_level", logging.INFO) + log_level = self.serena_config.gui_log_window_level if Logger.root.level > log_level: log.info(f"Root logger level is higher than GUI log level; changing the root logger level to {log_level}") Logger.root.setLevel(log_level) self._gui_log_handler = GuiLogViewerHandler(GuiLogViewer(title="Serena Logs"), level=log_level, format_string=LOG_FORMAT) Logger.root.addHandler(self._gui_log_handler) - log.info( - f"Starting serena server for project {project_file_path} (language={self.language}, root={self.project_root}); " - f"process id={os.getpid()}, parent process id={os.getppid()}" - ) - - # create and start the language server instance - self._config = MultilspyConfig(code_language=self.language) - self._ls_logger = MultilspyLogger() - self.language_server = SyncLanguageServer.create(self._config, self._ls_logger, self.project_root) + log.info(f"Starting Serena server (process id={os.getpid()}, parent process id={os.getppid()})") self.prompt_factory = PromptFactory() - self.symbol_manager = SymbolManager(self.language_server, self) - self.memories_manager = MemoriesManager(os.path.join(self.get_serena_managed_dir(), "memories")) - self.lines_read = LinesRead() + self._project_activation_callback = project_activation_callback + + # project-specific instances, which will be initialized upon project activation + self.project_config: ProjectConfig | None = None + self.language_server: SyncLanguageServer | None = None + self.symbol_manager: SymbolManager | None = None + self.memories_manager: MemoriesManager | None = None + self.lines_read: LinesRead | None = None # find all tool classes and instantiate them - excluded_tools = project_config.get("excluded_tools", []) self._all_tools: dict[type[Tool], Tool] = {} - self.tools: dict[type[Tool], Tool] = {} for tool_class in iter_tool_classes(): tool_instance = tool_class(self) + if not self.serena_config.enable_project_activation: + if tool_class in (GetActiveProjectTool, ActivateProjectTool): + log.info(f"Excluding tool '{tool_instance.get_name()}' because project activation is disabled in configuration") + continue self._all_tools[tool_class] = tool_instance - if (tool_name := tool_class.get_name()) in excluded_tools: - log.info(f"Skipping tool {tool_name} because it is in the exclude list") - continue - self.tools[tool_class] = tool_instance - log.info(f"Loaded tools: {', '.join([tool.get_name() for tool in self.tools.values()])}") + self._active_tools = dict(self._all_tools) + log.info(f"Loaded tools ({len(self._all_tools)}): {', '.join([tool.get_name() for tool in self._all_tools.values()])}") # If GUI log window is enabled, set the tool names for highlighting if self._gui_log_handler is not None: - tool_names = [tool.get_name() for tool in self.tools.values()] + tool_names = [tool.get_name() for tool in self._active_tools.values()] self._gui_log_handler.log_viewer.set_tool_names(tool_names) - # start the language server if requested - if self._start_language_server: - log.info("Starting the language server ...") - self.language_server.start() - if not self.language_server.is_running(): - raise RuntimeError(f"Failed to start the language server for {self.language} and project {self.project_root}") + + # activate a project configuration (if provided or if there is only a single project available) + project_config: ProjectConfig | None = None + if project_file_path is not None: + if not os.path.exists(project_file_path): + raise FileNotFoundError(f"Project file not found: {project_file_path}") + project_config = ProjectConfig.from_yml(project_file_path) + else: + match len(self.serena_config.projects): + case 0: + raise RuntimeError(f"No projects found in {SerenaConfig.CONFIG_FILE} and no project file specified.") + case 1: + project_config = self.serena_config.read_project_configuration(self.serena_config.project_names[0]) + if project_config is not None: + self.activate_project(project_config) + else: + if not self.serena_config.enable_project_activation: + raise ValueError("Tool-based project activation is disabled in the configuration but no project file was provided.") + + def get_exposed_tools(self) -> list["Tool"]: + """ + :return: the list of tools that are to be exposed/registered in the client + """ + if self.serena_config.enable_project_activation: + # With project activation, we must expose all tools and handle tool activation within Serena + # (because clients to not react to changed tools) + return list(self._all_tools.values()) + else: + return list(self._active_tools.values()) + + def activate_project(self, project_config: ProjectConfig) -> None: + log.info(f"Activating {project_config}") + self.project_config = project_config + + # handle project-specific tool exclusions (if any) + if self.project_config.excluded_tools: + self._active_tools = { + key: tool for key, tool in self._all_tools.items() if tool.get_name() not in project_config.excluded_tools + } + log.info(f"Active tools after exclusions ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}") + else: + self._active_tools = dict(self._all_tools) + + # start the language server + self.reset_language_server() + assert self.language_server is not None + + # initialize project-specific instances + self.symbol_manager = SymbolManager(self.language_server, self) + self.memories_manager = MemoriesManager(os.path.join(self.project_config.get_serena_managed_dir(), "memories")) + self.lines_read = LinesRead() + + if self._project_activation_callback is not None: + self._project_activation_callback() + + def get_active_tool_names(self) -> list[str]: + """ + :return: the list of names of the active tools for the current project + """ + return sorted([tool.get_name() for tool in self._active_tools.values()]) + + def is_language_server_running(self) -> bool: + return self.language_server is not None and self.language_server.is_running() def reset_language_server(self) -> None: - self.language_server.stop() + """ + Starts/resets the language server for the current project + """ + # stop the language server if it is running + if self.is_language_server_running(): + log.info("Stopping the language server ...") + assert self.language_server is not None + self.language_server.stop() + self.language_server = None + + # instantiate and start the language server + assert self.project_config is not None + multilspy_config = MultilspyConfig(code_language=self.project_config.language) + ls_logger = MultilspyLogger() + self.language_server = SyncLanguageServer.create(multilspy_config, ls_logger, self.project_config.project_root) self.language_server.start() if not self.language_server.is_running(): - raise RuntimeError(f"Failed to start (reset) the language server for {self.language} and project {self.project_root}") + raise RuntimeError(f"Failed to start the language server for {self.project_config}") def get_tool(self, tool_class: type[TTool]) -> TTool: return self._all_tools[tool_class] # type: ignore def print_tool_overview(self) -> None: - _print_tool_overview(self.tools.values()) - - def get_serena_managed_dir(self) -> str: - return os.path.join(self.project_root, ".serena") + _print_tool_overview(self._active_tools.values()) def mark_file_modified(self, relativ_path: str) -> None: + assert self.lines_read is not None self.lines_read.invalidate_lines_read(relativ_path) def __del__(self) -> None: @@ -150,8 +267,9 @@ class SerenaAgent: if not hasattr(self, "_is_initialized"): return log.info("SerenaAgent is shutting down ...") - if self._start_language_server: + if self.is_language_server_running(): log.info("Stopping the language server ...") + assert self.language_server is not None self.language_server.stop() if self._gui_log_handler: log.info("Stopping the GUI log window ...") @@ -195,14 +313,17 @@ class Component(ABC): @property def language_server(self) -> SyncLanguageServer: + assert self.agent.language_server is not None return self.agent.language_server @property def project_root(self) -> str: - return self.agent.project_root + assert self.project_config is not None + return self.project_config.project_root @property - def project_config(self) -> dict[str, Any]: + def project_config(self) -> ProjectConfig: + assert self.agent.project_config is not None return self.agent.project_config @property @@ -211,12 +332,19 @@ class Component(ABC): @property def memories_manager(self) -> MemoriesManager: + assert self.agent.memories_manager is not None return self.agent.memories_manager @property def symbol_manager(self) -> SymbolManager: + assert self.agent.symbol_manager is not None return self.agent.symbol_manager + @property + def lines_read(self) -> LinesRead: + assert self.agent.lines_read is not None + return self.agent.lines_read + _DEFAULT_MAX_ANSWER_LENGTH = int(2e5) @@ -286,25 +414,50 @@ class Tool(Component): """ Applies the tool with the given arguments """ - if not self.language_server.is_running(): - self.agent.reset_language_server() - apply_fn = self.get_apply_fn() + if log_call: self._log_tool_application(inspect.currentframe()) + try: + # check whether the tool requires an active project and language server + if not isinstance(self, ToolMarkerDoesNotRequireActiveProject): + if self.agent.project_config is None: + return ( + "Error: No active project. Ask to user to select a project from this list: " + + f"{self.agent.serena_config.project_names}" + ) + if not self.agent.is_language_server_running(): + log.info("Language server is not running. Starting it ...") + self.agent.reset_language_server() + + # check whether the tool is enabled + if self.agent.project_config is not None and self.get_name() in self.agent.project_config.excluded_tools: + return ( + f"Error: Tool '{self.get_name()}' is disabled for the active project ('{self.project_config.project_name}'); " + f"active tools: {self.agent.get_active_tool_names()}" + ) + + # apply the actual tool result = apply_fn(**kwargs) + except Exception as e: if not catch_exceptions: raise msg = f"Error executing tool: {e}\n{traceback.format_exc()}" log.error(f"Error executing tool: {e}", exc_info=e) result = msg + if log_call: log.info(f"Result: {result}") + return result +class ToolMarkerDoesNotRequireActiveProject: + pass + + class ReadFileTool(Tool): """ Reads a file within the project directory. @@ -331,7 +484,7 @@ class ReadFileTool(Tool): if end_line is None: result_lines = result_lines[start_line:] else: - self.agent.lines_read.add_lines_read(relative_path, (start_line, end_line)) + self.lines_read.add_lines_read(relative_path, (start_line, end_line)) result_lines = result_lines[start_line : end_line + 1] result = "\n".join(result_lines) @@ -379,7 +532,7 @@ class ListDirTool(Tool): os.path.join(self.project_root, relative_path), relative_to=self.project_root, recursive=recursive, - ignored_dirs=self.project_config["ignored_dirs"], + ignored_dirs=self.project_config.ignored_dirs, ) result = json.dumps({"dirs": dirs, "files": files}) return self._limit_length(result, max_answer_chars) @@ -661,7 +814,7 @@ class DeleteLinesTool(Tool): :param start_line: the 0-based index of the first line to be deleted :param end_line: the 0-based index of the last line to be deleted """ - if not self.agent.lines_read.were_lines_read(relative_path, (start_line, end_line)): + if not self.lines_read.were_lines_read(relative_path, (start_line, end_line)): read_lines_tool = self.agent.get_tool(ReadFileTool) return f"Error: Must call `{read_lines_tool.get_name()}` first to read exactly the affected lines." self.symbol_manager.delete_lines(relative_path, start_line, end_line) @@ -985,6 +1138,43 @@ class ExecuteShellCommandTool(Tool): return self._limit_length(result, max_answer_chars) +class GetActiveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): + """ + Gets the name of the currently active project (if any) and lists existing projects + """ + + def apply( + self, + ) -> str: + """ + Gets the name of the currently active project (if any) and returns the list of all available projects. + To change the current project, use the `activate_project` tool. + + :return: an object containing the name of the currently activated project (if any) and the list of all available projects + """ + active_project = None if self.agent.project_config is None else self.agent.project_config.project_name + return json.dumps({"active_project": active_project, "available_projects": self.agent.serena_config.project_names}) + + +class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): + """ + Activates a project by name. + """ + + def apply(self, project_name: str) -> str: + """ + Activates the project with the given name + + :param project_name: the name of the project to activate + """ + try: + project_config = self.agent.serena_config.read_project_configuration(project_name) + except ValueError as e: + return str(e) + self.agent.activate_project(project_config) + return SUCCESS_RESULT + + def iter_tool_classes() -> Generator[type[Tool], None, None]: return iter_subclasses(Tool) diff --git a/src/serena/agno.py b/src/serena/agno.py index 80f3316..09893f2 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -81,7 +81,7 @@ _patch_gemini_schema_conversion() class SerenaAgnoToolkit(Toolkit): def __init__(self, serena_agent: SerenaAgent): super().__init__("Serena") - for tool in serena_agent.tools.values(): + for tool in serena_agent.get_exposed_tools(): self.functions[tool.get_name()] = self._create_agno_function(tool) log.info("Agno agent functions: %s", list(self.functions.keys())) @@ -128,22 +128,25 @@ class SerenaAgnoAgentProvider: parser = argparse.ArgumentParser(description="Serena coding assistant") parser.add_argument( - "--project-file", required=True, help="Path to the project file, either absolute or relative to the root directory" + "--project-file", required=False, help="Path to the project file, either absolute or relative to the root directory" ) args = parser.parse_args() - project_file = Path(args.project_file).resolve() - # If project file path is relative, make it absolute by joining with project root - if not project_file.is_absolute(): - # Get the project root directory (parent of scripts directory) - project_root = Path(serena_root_path()) - project_file = project_root / args.project_file + if args.project_file: + project_file = Path(args.project_file).resolve() + # If project file path is relative, make it absolute by joining with project root + if not project_file.is_absolute(): + # Get the project root directory (parent of scripts directory) + project_root = Path(serena_root_path()) + project_file = project_root / args.project_file - # Ensure the path is normalized and absolute - project_file = project_file.resolve() + # Ensure the path is normalized and absolute + project_file = str(project_file.resolve()) + else: + project_file = None with LogTime("Loading Serena agent"): - serena_agent = SerenaAgent(str(project_file), start_language_server=True) + serena_agent = SerenaAgent(project_file) # Even though we don't want to keep history between sessions, # for agno-ui to work as a conversation, we use a persistent storage on disk. diff --git a/src/serena/gui_log_viewer.py b/src/serena/gui_log_viewer.py index b2bdebe..aa5dc51 100644 --- a/src/serena/gui_log_viewer.py +++ b/src/serena/gui_log_viewer.py @@ -301,6 +301,13 @@ class GuiLogViewerHandler(logging.Handler): if not self.log_viewer.running: self.log_viewer.start() + @classmethod + def is_instance_registered(cls) -> bool: + for h in logging.Logger.root.handlers: + if isinstance(h, cls): + return True + return False + def emit(self, record): """ Emit a log record to the ThreadedLogViewer. diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 871aa96..2300394 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -2,6 +2,7 @@ The Serena Model Context Protocol (MCP) Server """ +import time import sys from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -13,8 +14,10 @@ from mcp.server.fastmcp.server import FastMCP, Settings from mcp.server.fastmcp.tools.base import Tool as MCPTool from mcp.server.fastmcp.utilities.func_metadata import func_metadata from sensai.util import logging +from sensai.util.helper import mark_used from serena.agent import SerenaAgent, Tool +from serena.gui_log_viewer import GuiLogViewerHandler log = logging.getLogger(__name__) LOG_FORMAT = "%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s" @@ -41,7 +44,6 @@ class SerenaMCPRequestContext: def make_tool( tool: Tool, ) -> MCPTool: - """Create a Tool from a function.""" func_name = tool.get_name() apply_fn = getattr(tool, "apply") @@ -68,42 +70,55 @@ def make_tool( ) -_SERENA_AGENT_REGISTRY: dict[str, SerenaAgent] = {} -"""Map of project file paths to their corresponding SerenaAgent instances.""" - - -def _get_create_serena_agent(project_file_path: str) -> SerenaAgent: - if project_file_path not in _SERENA_AGENT_REGISTRY: - log.info(f"Creating new SerenaAgent for project file: {project_file_path}") - _SERENA_AGENT_REGISTRY[project_file_path] = SerenaAgent(project_file_path, start_language_server=True) - else: - log.info(f"Reusing existing SerenaAgent for project file: {project_file_path}") - return _SERENA_AGENT_REGISTRY[project_file_path] - - def create_mcp_server() -> FastMCP: argv = sys.argv[1:] - if len(argv) != 1: - print("\nUsage: mcp_server <.yml project file>", file=sys.stderr) - sys.exit(1) - project_file_path = argv[0] + if (len(argv) == 1 and argv[0] == "--help") or len(argv) > 1: + print("\nUsage: mcp_server [.yml project file]", file=sys.stderr) + sys.exit(0) - agent = _get_create_serena_agent(project_file_path) + mcp: FastMCP | None = None + + def update_tools() -> None: + """Update the tools in the MCP server.""" + # Tools may change as a result of project activation. + # NOTE: While we could pass updated tool information on to the MCP server via the callback, Claude Desktop does not, + # unfortunately, query for changed tools. It only queries for changed resources and prompts regularly, + # so we need to register all tools at startup, unfortunately. + nonlocal mcp + tools = agent.get_exposed_tools() + if mcp is not None: + mcp._tool_manager._tools = {} + for tool in tools: + # noinspection PyProtectedMember + mcp._tool_manager._tools[tool.get_name()] = make_tool(tool) + + project_file_path = argv[0] if len(argv) == 1 else None + try: + agent = SerenaAgent( + project_file_path, + # Callback disabled for the time being (see above) + # project_activation_callback=update_tools + ) + except Exception as e: + # allow any initialization errors to be viewed for some time in the GUI before exiting + if GuiLogViewerHandler.is_instance_registered(): + log.error(f"Failed to initialize agent: {e}", exc_info=e) + time.sleep(60) + raise @asynccontextmanager async def server_lifespan(mcp_server: FastMCP) -> AsyncIterator[None]: """Manage server startup and shutdown lifecycle.""" - try: - yield - finally: - for agent in _SERENA_AGENT_REGISTRY.values(): - agent.language_server.stop() + nonlocal agent + mark_used(mcp_server) + yield mcp_settings = Settings(lifespan=server_lifespan) mcp = FastMCP(**mcp_settings.model_dump()) - for tool in agent.tools.values(): - mcp._tool_manager._tools[tool.get_name()] = make_tool(tool) + + update_tools() + return mcp From 8e2013d8c15bd11e10621c62c94c3781606ba295 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Apr 2025 21:42:49 +0200 Subject: [PATCH 06/11] Add mechanism for showing fatal errors in GUI upon initialization failure --- pyproject.toml | 1 + src/serena/agno.py | 7 +++++- src/serena/gui_log_viewer.py | 41 +++++++++++++++++++++++++++--------- src/serena/mcp.py | 8 ++----- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b48ffa..a6de391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,6 +233,7 @@ ignore = [ "W293", "B009", "SIM103", # forbids multiple returns + "SIM110", # requires use of any(...) instead of for-loop ] unfixable = [ "F841", diff --git a/src/serena/agno.py b/src/serena/agno.py index 09893f2..0483293 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -16,6 +16,7 @@ from sensai.util.logging import LogTime from serena import serena_root_path from serena.agent import SerenaAgent, Tool +from serena.gui_log_viewer import show_fatal_exception log = logging.getLogger(__name__) @@ -146,7 +147,11 @@ class SerenaAgnoAgentProvider: project_file = None with LogTime("Loading Serena agent"): - serena_agent = SerenaAgent(project_file) + try: + serena_agent = SerenaAgent(project_file) + except Exception as e: + show_fatal_exception(e) + raise # Even though we don't want to keep history between sessions, # for agno-ui to work as a conversation, we use a persistent storage on disk. diff --git a/src/serena/gui_log_viewer.py b/src/serena/gui_log_viewer.py index aa5dc51..dfe554a 100644 --- a/src/serena/gui_log_viewer.py +++ b/src/serena/gui_log_viewer.py @@ -3,10 +3,14 @@ import logging import queue import sys import threading +import time import tkinter as tk +import traceback from enum import Enum, auto from pathlib import Path +log = logging.getLogger(__name__) + class LogLevel(Enum): DEBUG = auto() @@ -57,8 +61,7 @@ class GuiLogViewer: """Start the log viewer in a separate thread.""" if not self.running: self.print_status("Starting thread") - self.running = True - self.log_thread = threading.Thread(target=self._run_gui) + self.log_thread = threading.Thread(target=self.run_gui) self.log_thread.daemon = True self.log_thread.start() return True @@ -67,7 +70,6 @@ class GuiLogViewer: def stop(self): """Stop the log viewer.""" if self.running: - self.running = False # Add a sentinel value to the queue to signal the GUI to exit self.message_queue.put(None) return True @@ -91,11 +93,7 @@ class GuiLogViewer: message (str): The log message to display """ - if self.running: - # Add the original message so we can determine log level correctly - self.message_queue.put(message) - return True - return False + self.message_queue.put(message) def _determine_log_level(self, message): """ @@ -199,8 +197,9 @@ class GuiLogViewer: if self.running: self.root.after(100, self._process_queue) - def _run_gui(self): - """Run the GUI in a separate thread.""" + def run_gui(self): + """Run the GUI""" + self.running = True try: self.root = tk.Tk() self.root.title(self.title) @@ -346,3 +345,25 @@ class GuiLogViewerHandler(logging.Handler): """ if self.log_viewer.running: self.log_viewer.stop() + + +def show_fatal_exception(e: Exception, duration_secs: int = 60): + """ + Makes sure the given exception is shown in the GUI log viewer, + either an existing instance or a new one. + + :param e: the exception to display + :param duration_secs: the duration for which to display the error before + terminating the program for the case where the log viewer is already present + in a daemon thread which will terminate when the program does + """ + if GuiLogViewerHandler.is_instance_registered(): + # show in existing daemon thread, waiting for the given duration + log.error(f"Fatal error: {e}", exc_info=e) + time.sleep(duration_secs) + else: + # show in new window in main thread (user must close it) + log_viewer = GuiLogViewer() + exc_info = "".join(traceback.format_exception(type(e), e, e.__traceback__)) + log_viewer.add_log(f"ERROR Fatal exception: {e}\n{exc_info}") + log_viewer.run_gui() diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 2300394..d55a5b1 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -2,7 +2,6 @@ The Serena Model Context Protocol (MCP) Server """ -import time import sys from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -17,7 +16,7 @@ from sensai.util import logging from sensai.util.helper import mark_used from serena.agent import SerenaAgent, Tool -from serena.gui_log_viewer import GuiLogViewerHandler +from serena.gui_log_viewer import show_fatal_exception log = logging.getLogger(__name__) LOG_FORMAT = "%(levelname)-5s %(asctime)-15s %(name)s:%(funcName)s:%(lineno)d - %(message)s" @@ -101,10 +100,7 @@ def create_mcp_server() -> FastMCP: # project_activation_callback=update_tools ) except Exception as e: - # allow any initialization errors to be viewed for some time in the GUI before exiting - if GuiLogViewerHandler.is_instance_registered(): - log.error(f"Failed to initialize agent: {e}", exc_info=e) - time.sleep(60) + show_fatal_exception(e) raise @asynccontextmanager From 9c1c2a4d2aa7c641849fb1aab2a0b637de444b82 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Apr 2025 21:47:13 +0200 Subject: [PATCH 07/11] Fix log level highlighting not working when tools are registered --- src/serena/gui_log_viewer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serena/gui_log_viewer.py b/src/serena/gui_log_viewer.py index dfe554a..8c55251 100644 --- a/src/serena/gui_log_viewer.py +++ b/src/serena/gui_log_viewer.py @@ -146,7 +146,7 @@ class GuiLogViewer: start_index = self.text_widget.index("end-1c") # Insert the message - self.text_widget.insert(tk.END, message + "\n") + self.text_widget.insert(tk.END, message + "\n", log_level.name) # Convert start index to line/char format line, char = map(int, start_index.split(".")) From 8f9415ec3c7234fe54a5bef4b87c91046a574e12 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 6 Apr 2025 23:22:06 +0200 Subject: [PATCH 08/11] Support default project file location via specification of project root directory when specifying list of projects in serena_config.yml: /path/to/project/.serena/project.yml --- myproject.template.yml | 3 ++- pyproject.toml | 1 + src/serena/agent.py | 51 +++++++++++++++++++++++++----------------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/myproject.template.yml b/myproject.template.yml index d1c4cce..907f231 100644 --- a/myproject.template.yml +++ b/myproject.template.yml @@ -1,4 +1,5 @@ -# absolute path to the project +# absolute path to the project you want Serena to work on (where all the source code, etc. is located) +# This is optional if this file is placed in the project directory under `.serena/project.yml`. project_root: /path/to/project # language of the project (csharp, python, rust, java, typescript, javascript, go, or ruby) diff --git a/pyproject.toml b/pyproject.toml index a6de391..be7606a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,7 @@ ignore = [ "B009", "SIM103", # forbids multiple returns "SIM110", # requires use of any(...) instead of for-loop + "G001", # forbids str.format in log statements ] unfixable = [ "F841", diff --git a/src/serena/agent.py b/src/serena/agent.py index cc7b4fe..d354e8b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -39,11 +39,14 @@ SUCCESS_RESULT = "OK" class ProjectConfig(ToStringMixin): SERENA_MANAGED_DIR = ".serena" + SERENA_DEFAULT_PROJECT_FILE = "project.yml" - def __init__(self, config_dict: dict[str, Any], project_name: str): + def __init__(self, config_dict: dict[str, Any], project_name: str, project_root: Path | None = None): self.project_name: str = project_name self.language: Language = Language(config_dict["language"]) - self.project_root: str = str(Path(config_dict["project_root"]).resolve()) + if project_root is None: + project_root = Path(config_dict["project_root"]) + self.project_root: str = str(project_root.resolve()) self.ignored_dirs: list[str] = config_dict.get("ignored_dirs", []) self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) @@ -51,14 +54,16 @@ class ProjectConfig(ToStringMixin): return ["ignored_dirs", "excluded_tools"] @classmethod - def project_name_from_yml_path(cls, yml_file_path: str | Path) -> str: - return Path(yml_file_path).stem - - @classmethod - def from_yml(cls, yml_path: str | Path) -> Self: + def from_yml(cls, yml_path: Path) -> Self: with open(yml_path, encoding="utf-8") as f: config_dict = yaml.safe_load(f) - return cls(config_dict, cls.project_name_from_yml_path(yml_path)) + if yml_path.parent.name == cls.SERENA_MANAGED_DIR: + project_root = yml_path.parent.parent + project_name = project_root.name + else: + project_root = None + project_name = yml_path.stem + return cls(config_dict, project_name=project_name, project_root=project_root) def get_serena_managed_dir(self) -> str: return os.path.join(self.project_root, self.SERENA_MANAGED_DIR) @@ -79,23 +84,28 @@ class SerenaConfig: with open(config_file, encoding="utf-8") as f: config_yaml = yaml.safe_load(f) - self.projects: dict[str, Path] = {} - for project_yaml_path in config_yaml["projects"]: - project_yaml_path = Path(project_yaml_path) - if not project_yaml_path.is_absolute(): - project_yaml_path = Path(serena_root_path()) / project_yaml_path - project_name = project_yaml_path.stem - self.projects[project_name] = project_yaml_path + # read projects + self.projects: dict[str, ProjectConfig] = {} + for project_config_path in config_yaml["projects"]: + project_config_path = Path(project_config_path) + if not project_config_path.is_absolute(): + project_config_path = Path(serena_root_path()) / project_config_path + if project_config_path.is_dir(): # assume project file in default location + project_config_path = project_config_path / ProjectConfig.SERENA_MANAGED_DIR / ProjectConfig.SERENA_DEFAULT_PROJECT_FILE + if not project_config_path.is_file(): + raise FileNotFoundError(f"Project file not found: {project_config_path}") + project_config = ProjectConfig.from_yml(project_config_path) + self.projects[project_config.project_name] = project_config self.project_names = list(self.projects.keys()) self.gui_log_window_enabled = config_yaml.get("gui_log_window", True) self.gui_log_window_level = config_yaml.get("gui_log_level", logging.INFO) self.enable_project_activation = config_yaml.get("enable_project_activation", True) - def read_project_configuration(self, project_name: str) -> ProjectConfig: + def get_project_configuration(self, project_name: str) -> ProjectConfig: if project_name not in self.projects: raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}") - return ProjectConfig.from_yml(self.projects[project_name]) + return self.projects[project_name] class LinesRead: @@ -139,6 +149,7 @@ class SerenaAgent: Logger.root.addHandler(self._gui_log_handler) log.info(f"Starting Serena server (process id={os.getpid()}, parent process id={os.getppid()})") + log.info("Available projects: {}".format(", ".join(self.serena_config.project_names))) self.prompt_factory = PromptFactory() self._project_activation_callback = project_activation_callback @@ -172,13 +183,13 @@ class SerenaAgent: if project_file_path is not None: if not os.path.exists(project_file_path): raise FileNotFoundError(f"Project file not found: {project_file_path}") - project_config = ProjectConfig.from_yml(project_file_path) + project_config = ProjectConfig.from_yml(Path(project_file_path)) else: match len(self.serena_config.projects): case 0: raise RuntimeError(f"No projects found in {SerenaConfig.CONFIG_FILE} and no project file specified.") case 1: - project_config = self.serena_config.read_project_configuration(self.serena_config.project_names[0]) + project_config = self.serena_config.get_project_configuration(self.serena_config.project_names[0]) if project_config is not None: self.activate_project(project_config) else: @@ -1168,7 +1179,7 @@ class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): :param project_name: the name of the project to activate """ try: - project_config = self.agent.serena_config.read_project_configuration(project_name) + project_config = self.agent.serena_config.get_project_configuration(project_name) except ValueError as e: return str(e) self.agent.activate_project(project_config) From a4a8c19f6cfd6bd2ba0a4ce0879f0e77bbd46211 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Apr 2025 00:04:44 +0200 Subject: [PATCH 09/11] ProjectConfig: Remove to-string exclusions --- src/serena/agent.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index d354e8b..ac459bc 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -50,9 +50,6 @@ class ProjectConfig(ToStringMixin): self.ignored_dirs: list[str] = config_dict.get("ignored_dirs", []) self.excluded_tools: set[str] = set(config_dict.get("excluded_tools", [])) - def _tostring_excludes(self) -> list[str]: - return ["ignored_dirs", "excluded_tools"] - @classmethod def from_yml(cls, yml_path: Path) -> Self: with open(yml_path, encoding="utf-8") as f: From fdc13403de5bc4cde382c154c22551788966befa Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Apr 2025 00:32:28 +0200 Subject: [PATCH 10/11] Update change log --- CHANGELOG.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7128943..eb3d0cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ -# Changelog +# 2025-04-07 + +* Allow Serena to switch between projects (project activation) + * Add central Serena configuration in `serena_config.yml`, which + * contains the list of available projects + * allows to configure whether project activation is enabled + * now contains the GUI logging configuration (project configurations no longer do) + * Add new tools `activate_project` and `get_active_project` + * Providing a project configuration file in the launch parameters is now optional +* Logging: + * Improve error reporting in case of initialization failure: + open a new GUI log window showing the error or ensure that the existing log window remains visible for some time +* Language servers: + * Fix C# language server initialization issue when the project path contains spaces +* Agno: + * Fix Agno reloading mechanism causing failures when initializing the sqlite memory database #8 + * Fix Serena GUI log window not capturing logs after initialization + +# 2025-04-01 + +Initial public version + -## 01.04.2025: Initial Release \ No newline at end of file From 34fc91021089746c12b0f389a86918139544b9d3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 7 Apr 2025 01:07:35 +0200 Subject: [PATCH 11/11] Improve version reporting, adding git status if available --- pyproject.toml | 1 + src/serena/__init__.py | 21 ++++++++++++++++++++- src/serena/agent.py | 4 ++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be7606a..bf6bdb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,6 +235,7 @@ ignore = [ "SIM103", # forbids multiple returns "SIM110", # requires use of any(...) instead of for-loop "G001", # forbids str.format in log statements + "E722", # forbids unspecific except clause ] unfixable = [ "F841", diff --git a/src/serena/__init__.py b/src/serena/__init__.py index 9a6ae1b..bcb2bdb 100644 --- a/src/serena/__init__.py +++ b/src/serena/__init__.py @@ -1,7 +1,26 @@ -__version__ = "0.1.0-dev1" +__version__ = "2025-04-07" def serena_root_path() -> str: from pathlib import Path return str(Path(__file__).parent.parent.parent.absolute()) + + +def serena_version() -> str: + """ + :return: the version of the package, including git status if available. + """ + version = __version__ + try: + from sensai.util.git import git_status + from sensai.util.logging import LoggingDisabledContext + + with LoggingDisabledContext(): + git_status = git_status() + version += f"-{git_status.commit[:8]}" + if not git_status.is_clean: + version += "-dirty" + except: + pass + return version diff --git a/src/serena/agent.py b/src/serena/agent.py index 78acc7a..e84b7c0 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -22,7 +22,7 @@ from multilspy import SyncLanguageServer from multilspy.multilspy_config import Language, MultilspyConfig from multilspy.multilspy_logger import MultilspyLogger from multilspy.multilspy_types import SymbolKind -from serena import __version__, serena_root_path +from serena import serena_root_path, serena_version from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler from serena.llm.prompt_factory import PromptFactory from serena.symbol import SymbolLocation, SymbolManager @@ -145,7 +145,7 @@ class SerenaAgent: self._gui_log_handler = GuiLogViewerHandler(GuiLogViewer(title="Serena Logs"), level=log_level, format_string=LOG_FORMAT) Logger.root.addHandler(self._gui_log_handler) - log.info(f"Starting Serena server (version={__version__}, process id={os.getpid()}, parent process id={os.getppid()})") + log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})") log.info("Available projects: {}".format(", ".join(self.serena_config.project_names))) self.prompt_factory = PromptFactory()