Use a synchronized task queue for agent tasks (thread pool with 1 worker),

which we also use to start the language server in the background

src/serena/agent.py

@@ -14,6 +14,7 @@ import webbrowser
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Callable, Generator, Iterable, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor
 from copy import copy, deepcopy
 from dataclasses import asdict, dataclass, field
 from fnmatch import fnmatch
@@ -30,7 +31,7 @@ from overrides import override
 from pathspec import PathSpec
 from ruamel.yaml.comments import CommentedMap
 from sensai.util import logging
-from sensai.util.logging import LOG_DEFAULT_FORMAT, FallbackHandler
+from sensai.util.logging import LOG_DEFAULT_FORMAT, FallbackHandler, LogTime
 from sensai.util.string import ToStringMixin, dict_string
 
 from serena import serena_version
@@ -50,7 +51,6 @@ from serena.util.file_system import GitignoreParser, match_path, scan_directory
 from serena.util.general import load_yaml, save_yaml
 from serena.util.inspection import determine_programming_language_composition, iter_subclasses
 from serena.util.shell import execute_shell_command
-from serena.util.thread import ExecutionResult, execute_with_timeout
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language, LanguageServerConfig
 from solidlsp.ls_logger import LanguageServerLogger
@@ -787,6 +787,10 @@ class SerenaAgent:
         log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})")
         log.info("Available projects: {}".format(", ".join(self.serena_config.project_names)))
 
+        # create executor for starting the language server and running tools in another thread
+        # This executor is used to achieve linear task execution, so it is important to use a single-threaded executor.
+        self._task_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SerenaAgentExecutor")
+
         # Initialize the prompt factory
         self.prompt_factory = SerenaPromptFactory()
         self._project_activation_callback = project_activation_callback
@@ -949,21 +953,34 @@ class SerenaAgent:
 
         log.info(f"Active tools after all exclusions ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}")
 
+    def issue_task(self, task: Callable[[], Any]) -> Future:
+        """
+        Issue a task to the executor for asynchronous execution.
+        :param task: the task to execute
+        :return: a Future object representing the execution of the task
+        """
+        return self._task_executor.submit(task)
+
     def _activate_project(self, project: Project) -> None:
         log.info(f"Activating {project.project_name} at {project.project_root}")
         self._active_project = project
         self._update_active_tools()
 
-        # start the language server
-        self.reset_language_server()
-        assert self.language_server is not None
-        self.ignore_spec = self.language_server.get_ignore_spec()
+        def init_language_server() -> None:
+            # start the language server
+            with LogTime("Language server initialization", logger=log):
+                self.reset_language_server()
+                assert self.language_server is not None
+                self.ignore_spec = self.language_server.get_ignore_spec()
 
-        # initialize project-specific instances
-        log.debug(f"Initializing symbol and memories manager for {project.project_name} at {project.project_root}")
-        self.symbol_manager = SymbolManager(self.language_server, self)
-        self.memories_manager = MemoriesManagerMDFilesInProject(project.project_root)
-        self.lines_read = LinesRead()
+            # initialize project-specific instances (some of which depend on the language server)
+            log.debug(f"Initializing symbol and memories manager for {project.project_name} at {project.project_root}")
+            self.symbol_manager = SymbolManager(self.language_server, self)
+            self.memories_manager = MemoriesManagerMDFilesInProject(project.project_root)
+            self.lines_read = LinesRead()
+
+        # initialize the language server in the background
+        self.issue_task(init_language_server)
 
         if self._project_activation_callback is not None:
             self._project_activation_callback()
@@ -1323,57 +1340,56 @@ class Tool(Component, ToolInterface):
         """
         Applies the tool with the given arguments
         """
-        apply_fn = self.get_apply_fn()
 
-        try:
-            if not self.is_active():
-                return f"Error: Tool '{self.get_name_from_cls()}' is not active. Active tools: {self.agent.get_active_tool_names()}"
-        except Exception as e:
-            return f"RuntimeError while checking if tool {self.get_name_from_cls()} is active: {e}"
+        def task() -> str:
+            apply_fn = self.get_apply_fn()
 
-        if log_call:
-            self._log_tool_application(inspect.currentframe())
-        try:
-            # check whether the tool requires an active project and language server
-            if not isinstance(self, ToolMarkerDoesNotRequireActiveProject):
-                if self.agent._active_project is None:
-                    return (
-                        "Error: No active project. Ask to user to select a project from this list: "
-                        + f"{self.agent.serena_config.project_names}"
-                    )
-                if not self.agent.is_language_server_running():
-                    log.info("Language server is not running. Starting it ...")
-                    self.agent.reset_language_server()
+            try:
+                if not self.is_active():
+                    return f"Error: Tool '{self.get_name_from_cls()}' is not active. Active tools: {self.agent.get_active_tool_names()}"
+            except Exception as e:
+                return f"RuntimeError while checking if tool {self.get_name_from_cls()} is active: {e}"
 
-            # apply the actual tool with a timeout
-            execution_fn = lambda: apply_fn(**kwargs)
-            execution_result = execute_with_timeout(execution_fn, self.agent.serena_config.tool_timeout, self.get_name_from_cls())
-            if execution_result.status == ExecutionResult.Status.SUCCESS:
-                result = cast(str, execution_result.result_value)
-            else:
-                assert execution_result.exception is not None
-                raise execution_result.exception
+            if log_call:
+                self._log_tool_application(inspect.currentframe())
+            try:
+                # check whether the tool requires an active project and language server
+                if not isinstance(self, ToolMarkerDoesNotRequireActiveProject):
+                    if self.agent._active_project is None:
+                        return (
+                            "Error: No active project. Ask to user to select a project from this list: "
+                            + f"{self.agent.serena_config.project_names}"
+                        )
+                    if not self.agent.is_language_server_running():
+                        log.info("Language server is not running. Starting it ...")
+                        self.agent.reset_language_server()
 
-        except Exception as e:
-            if not catch_exceptions:
-                raise
-            msg = f"Error executing tool: {e}\n{traceback.format_exc()}"
-            log.error(
-                f"Error executing tool: {e}. "
-                f"Consider restarting the language server to solve this (especially, if it's a timeout of a symbolic operation)",
-                exc_info=e,
-            )
-            result = msg
+                # apply the actual tool
+                result = apply_fn(**kwargs)
 
-        if log_call:
-            log.info(f"Result: {result}")
+            except Exception as e:
+                if not catch_exceptions:
+                    raise
+                msg = f"Error executing tool: {e}\n{traceback.format_exc()}"
+                log.error(
+                    f"Error executing tool: {e}. "
+                    f"Consider restarting the language server to solve this (especially, if it's a timeout of a symbolic operation)",
+                    exc_info=e,
+                )
+                result = msg
 
-        try:
-            self.language_server.save_cache()
-        except Exception as e:
-            log.error(f"Error saving language server cache: {e}")
+            if log_call:
+                log.info(f"Result: {result}")
 
-        return result
+            try:
+                self.language_server.save_cache()
+            except Exception as e:
+                log.error(f"Error saving language server cache: {e}")
+
+            return result
+
+        future = self.agent.issue_task(task)
+        return future.result(timeout=self.agent.serena_config.tool_timeout)
 
 
 class RestartLanguageServerTool(Tool):

src/serena/mcp.py

@@ -220,6 +220,7 @@ class SerenaMCPFactorySingleProcess(SerenaMCPFactory):
     @asynccontextmanager
     async def server_lifespan(self, mcp_server: FastMCP) -> AsyncIterator[None]:
         self._set_mcp_tools(mcp_server)
+        log.info("MCP server lifetime setup complete")
         yield
 
 
@@ -427,6 +428,7 @@ class SerenaMCPFactoryWithProcessIsolation(SerenaMCPFactory):
         # Start monitoring task
         monitor_task = asyncio.create_task(monitor_global_shutdown())
 
+        log.info("MCP server lifetime setup complete")
         try:
             yield
         except (KeyboardInterrupt, SystemExit):