Merge pull request #313 from oraios/feature/enhanced_cli

Feature/enhanced cli

pyproject.toml

@@ -38,8 +38,13 @@ dependencies = [
 
 [project.scripts]
 serena-mcp-server = "serena.cli:start_mcp_server"
-index-project = "serena.cli:index_project"
+index-project = "serena.cli:index_project" # deprecated
 print-system-prompt = "serena.cli:print_system_prompt"
+mode = "serena.cli:mode"
+context = "serena.cli:context"
+project = "serena.cli:project"
+config = "serena.cli:config"
+help = "serena.cli:get_help"
 
 [project.license]
 text = "MIT"

src/serena/agent.py

@@ -21,7 +21,7 @@ from sensai.util.logging import LogTime
 from serena import serena_version
 from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.config.serena_config import SerenaConfig, ToolSet, get_serena_managed_dir
+from serena.config.serena_config import SerenaConfig, ToolSet, get_serena_managed_in_project_dir
 from serena.constants import (
     SERENA_LOG_FORMAT,
 )
@@ -62,7 +62,7 @@ class LinesRead:
 
 class MemoriesManager:
     def __init__(self, project_root: str):
-        self._memory_dir = Path(get_serena_managed_dir(project_root)) / "memories"
+        self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
         self._memory_dir.mkdir(parents=True, exist_ok=True)
 
     def _get_memory_file_path(self, name: str) -> Path:

src/serena/cli.py

@@ -1,237 +1,457 @@
 import os
+import shutil
+import subprocess
+import sys
 from pathlib import Path
-from typing import Literal
+from typing import Literal, cast
 
 import click
 from sensai.util import logging
 from tqdm import tqdm
 
 from serena.agent import SerenaAgent
-from serena.config.serena_config import SerenaConfig
-from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES
+from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
+from serena.config.serena_config import ProjectConfig, SerenaConfig
+from serena.constants import (
+    DEFAULT_CONTEXT,
+    DEFAULT_MODES,
+    SERENA_MANAGED_DIR_IN_HOME,
+    SERENAS_OWN_CONTEXT_YAMLS_DIR,
+    SERENAS_OWN_MODE_YAMLS_DIR,
+    USER_CONTEXT_YAMLS_DIR,
+    USER_MODE_YAMLS_DIR,
+)
 from serena.mcp import SerenaMCPFactorySingleProcess
 from serena.project import Project
+from solidlsp.ls_config import Language
 
 log = logging.getLogger(__name__)
 
+# --------------------- Utilities -------------------------------------
+
+
+def _open_in_editor(path: str) -> None:
+    """Open the given file in the system's default editor or viewer."""
+    editor = os.environ.get("EDITOR")
+    try:
+        if editor:
+            subprocess.run([editor, path], check=False)
+        elif sys.platform.startswith("win"):
+            os.startfile(path)
+        elif sys.platform == "darwin":
+            subprocess.run(["open", path], check=False)
+        else:
+            subprocess.run(["xdg-open", path], check=False)
+    except Exception as e:
+        print(f"Failed to open {path}: {e}")
+
 
 class ProjectType(click.ParamType):
+    """ParamType allowing either a project name or a path to a project directory."""
+
     name = "[PROJECT_NAME|PROJECT_PATH]"
 
-    def convert(self, value: str, param: click.Parameter | None, ctx: click.Context | None) -> str:
+    def convert(self, value: str, param, ctx) -> str:
         path = Path(value).resolve()
         if path.exists() and path.is_dir():
-            return str(path)  # Valid path
-        return value  # Assume it's a project name
+            return str(path)
+        return value
 
 
 PROJECT_TYPE = ProjectType()
 
 
-@click.command()
-@click.option(
-    "--project",
-    "project",
-    type=PROJECT_TYPE,
-    default=None,
-    help="Either an absolute path to the project directory or a name of an already registered project. "
-    "If the project passed here hasn't been registered yet, it will be registered automatically and can be activated by its name afterwards.",
-)
-# Keep --project-file for backwards compatibility
-@click.option(
-    "--project-file",
-    "project",  # Use same destination variable to avoid conflicts
-    type=PROJECT_TYPE,
-    default=None,
-    help="[DEPRECATED] Use --project instead.",
-)
-# Positional argument for backwards compatibility
-@click.argument(
-    "project_file_arg",
-    type=PROJECT_TYPE,
-    required=False,
-    default=None,
-    metavar="",  # don't display anything since it's deprecated
-)
-@click.option(
-    "--context",
-    type=str,
-    show_default=True,
-    default=DEFAULT_CONTEXT,
-    help="Context to use. This can be a name of a built-in context ('desktop-app', 'agent', 'ide-assistant') "
-    "or a path to a custom context YAML file.",
-)
-@click.option(
-    "--mode",
-    "modes",
-    type=str,
-    multiple=True,
-    default=DEFAULT_MODES,
-    show_default=True,
-    help="Mode(s) to use. This can be names of built-in modes ('planning', 'editing', 'one-shot', 'interactive') "
-    "or paths to custom mode YAML files. Can be specified multiple times to combine modes.",
-)
-@click.option(
-    "--transport",
-    type=click.Choice(["stdio", "sse"]),
-    default="stdio",
-    show_default=True,
-    help="Transport protocol. If you start the server yourself (as opposed to an MCP Client starting the server), sse is recommended.",
-)
-@click.option(
-    "--host",
-    type=str,
-    show_default=True,
-    default="0.0.0.0",
-    help="Host to bind to (for SSE transport).",
-)
-@click.option(
-    "--port",
-    type=int,
-    show_default=True,
-    default=8000,
-    help="Port to bind to (for SSE transport).",
-)
-@click.option(
-    "--enable-web-dashboard",
-    type=bool,
-    is_flag=False,
-    default=None,
-    help="Whether to enable the web dashboard. If not specified, will take the value from the serena configuration.",
-)
-@click.option(
-    "--enable-gui-log-window",
-    type=bool,
-    is_flag=False,
-    default=None,
-    help="Whether to enable the GUI log window. It currently does not work on macOS, and setting this to True will be ignored then. "
-    "If not specified, will take the value from the serena configuration.",
-)
-@click.option(
-    "--log-level",
-    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
-    default=None,
-    help="Log level for GUI, dashboard and other logging. If not specified, will take the value from the serena configuration.",
-)
-@click.option(
-    "--trace-lsp-communication",
-    type=bool,
-    is_flag=False,
-    default=None,
-    help="Whether to trace the communication between Serena and the language servers. This is useful for debugging language server issues.",
-)
-@click.option(
-    "--tool-timeout",
-    type=float,
-    default=None,
-    help="Timeout in seconds for tool execution. If not specified, will take the value from the serena configuration.",
-)
-def start_mcp_server(
-    project: str | None,
-    project_file_arg: str | None,
-    context: str = DEFAULT_CONTEXT,
-    modes: tuple[str, ...] = DEFAULT_MODES,
-    transport: Literal["stdio", "sse"] = "stdio",
-    host: str = "0.0.0.0",
-    port: int = 8000,
-    enable_web_dashboard: bool | None = None,
-    enable_gui_log_window: bool | None = None,
-    log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None = None,
-    trace_lsp_communication: bool | None = None,
-    tool_timeout: float | None = None,
-) -> None:
-    """Starts the Serena MCP server. By default, will not activate any project at startup.
-    If you want to start with an already active project, use --project to pass the project name or path.
-
-    Use --context to specify the execution environment and --mode to specify behavior mode(s).
-    The modes may be adjusted after startup (via the corresponding tool), but the context cannot be changed.
+class AutoRegisteringGroup(click.Group):
     """
-    # Prioritize the positional argument if provided
-    # This is for backward compatibility with the old CLI, should be removed in the future!
-    project_file = project_file_arg if project_file_arg is not None else project
+    A click.Group subclass that automatically registers any click.Command
+    attributes defined on the class into the group.
 
-    mcp_factory = SerenaMCPFactorySingleProcess(context=context, project=project_file)
-    mcp_server = mcp_factory.create_mcp_server(
-        host=host,
-        port=port,
-        modes=modes,
-        enable_web_dashboard=enable_web_dashboard,
-        enable_gui_log_window=enable_gui_log_window,
-        log_level=log_level,
-        trace_lsp_communication=trace_lsp_communication,
-        tool_timeout=tool_timeout,
+    After initialization, it inspects its own class for attributes that are
+    instances of click.Command (typically created via @click.command) and
+    calls self.add_command(cmd) on each. This lets you define your commands
+    as static methods on the subclass for IDE-friendly organization without
+    manual registration.
+    """
+
+    def __init__(self, name: str, help: str):
+        super().__init__(name=name, help=help)
+        # Scan class attributes for click.Command instances and register them.
+        for attr in dir(self.__class__):
+            cmd = getattr(self.__class__, attr)
+            if isinstance(cmd, click.Command):
+                self.add_command(cmd)
+
+
+class TopLevelCommands(AutoRegisteringGroup):
+    """Root CLI group containing the core Serena commands."""
+
+    def __init__(self):
+        super().__init__(name="serena", help="Serena CLI commands. You can run `<command> --help` for more info on each command.")
+
+    @staticmethod
+    @click.command("start-mcp-server", help="Starts the Serena MCP server.")
+    @click.option("--project", "project", type=PROJECT_TYPE, default=None, help="Path or name of project to activate at startup.")
+    @click.option("--project-file", "project", type=PROJECT_TYPE, default=None, help="[DEPRECATED] Use --project instead.")
+    @click.argument("project_file_arg", type=PROJECT_TYPE, required=False, default=None, metavar="")
+    @click.option(
+        "--context", type=str, default=DEFAULT_CONTEXT, show_default=True, help="Built-in context name or path to custom context YAML."
     )
+    @click.option(
+        "--mode",
+        "modes",
+        type=str,
+        multiple=True,
+        default=DEFAULT_MODES,
+        show_default=True,
+        help="Built-in mode names or paths to custom mode YAMLs.",
+    )
+    @click.option("--transport", type=click.Choice(["stdio", "sse"]), default="stdio", show_default=True, help="Transport protocol.")
+    @click.option("--host", type=str, default="0.0.0.0", show_default=True)
+    @click.option("--port", type=int, default=8000, show_default=True)
+    @click.option("--enable-web-dashboard", type=bool, is_flag=False, default=None, help="Override dashboard setting in config.")
+    @click.option("--enable-gui-log-window", type=bool, is_flag=False, default=None, help="Override GUI log window setting in config.")
+    @click.option(
+        "--log-level",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        default=None,
+        help="Override log level in config.",
+    )
+    @click.option("--trace-lsp-communication", type=bool, is_flag=False, default=None, help="Whether to trace LSP communication.")
+    @click.option("--tool-timeout", type=float, default=None, help="Override tool execution timeout in config.")
+    def start_mcp_server(
+        project: str | None,
+        project_file_arg: str | None,
+        context: str,
+        modes: tuple[str, ...],
+        transport: Literal["stdio", "sse"],
+        host: str,
+        port: int,
+        enable_web_dashboard: bool | None,
+        enable_gui_log_window: bool | None,
+        log_level: str | None,
+        trace_lsp_communication: bool | None,
+        tool_timeout: float | None,
+    ) -> None:
+        project_file = project_file_arg or project
+        factory = SerenaMCPFactorySingleProcess(context=context, project=project_file)
+        server = factory.create_mcp_server(
+            host=host,
+            port=port,
+            modes=modes,
+            enable_web_dashboard=enable_web_dashboard,
+            enable_gui_log_window=enable_gui_log_window,
+            log_level=log_level,
+            trace_lsp_communication=trace_lsp_communication,
+            tool_timeout=tool_timeout,
+        )
+        if project_file_arg:
+            log.warning(
+                "Positional project arg is deprecated; use --project instead. Used: %s",
+                project_file,
+            )
+        log.info("Starting MCP server …")
+        server.run(transport=transport)
 
-    # log after server creation such that the log appears in the GUI
-    if project_file_arg is not None:
-        log.warning(
-            "The positional argument for the project file path is deprecated and will be removed in the future! "
-            "Please pass the project file path via the `--project` option instead.\n"
-            f"Used path: {project_file}"
+    @staticmethod
+    @click.command("print-system-prompt", help="Print the system prompt for a project.")
+    @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False)
+    @click.option(
+        "--log-level",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        default="WARNING",
+        help="Log level for prompt generation.",
+    )
+    @click.option("--only-instructions", is_flag=True, help="Print only the initial instructions, without prefix/postfix.")
+    @click.option(
+        "--context", type=str, default=DEFAULT_CONTEXT, show_default=True, help="Built-in context name or path to custom context YAML."
+    )
+    @click.option(
+        "--mode",
+        "modes",
+        type=str,
+        multiple=True,
+        default=DEFAULT_MODES,
+        show_default=True,
+        help="Built-in mode names or paths to custom mode YAMLs.",
+    )
+    def print_system_prompt(project: str, log_level: str, only_instructions: bool, context: str, modes: tuple[str, ...]) -> None:
+        prefix = "You will receive access to Serena's symbolic tools. Below are instructions for using them, take them into account."
+        postfix = "You begin by acknowledging that you understood the above instructions and are ready to receive tasks."
+        from serena.tools.workflow_tools import InitialInstructionsTool
+
+        lvl = logging.getLevelNamesMapping()[log_level.upper()]
+        logging.configure(level=lvl)
+        context_instance = SerenaAgentContext.load(context)
+        mode_instances = [SerenaAgentMode.load(mode) for mode in modes]
+        agent = SerenaAgent(
+            project=os.path.abspath(project),
+            serena_config=SerenaConfig(web_dashboard=False, log_level=lvl),
+            context=context_instance,
+            modes=mode_instances,
+        )
+        tool = agent.get_tool(InitialInstructionsTool)
+        instr = tool.apply()
+        if only_instructions:
+            print(instr)
+        else:
+            print(f"{prefix}\n{instr}\n{postfix}")
+
+    @staticmethod
+    @click.command("help", help="Show help for Serena CLI commands.")
+    @click.pass_context
+    def help(ctx: click.Context) -> None:
+        # ctx.parent is the root invocation context
+        root = ctx.parent
+        click.echo(root.get_help())
+
+
+class ModeCommands(AutoRegisteringGroup):
+    """Group for 'mode' subcommands."""
+
+    def __init__(self):
+        super().__init__(name="mode", help="Manage Serena modes. You can run `mode <command> --help` for more info on each command.")
+
+    @staticmethod
+    @click.command("list", help="List available modes.")
+    def list():
+        mode_names = SerenaAgentMode.list_registered_mode_names()
+        max_len_name = max(len(name) for name in mode_names) if mode_names else 20
+        for name in mode_names:
+            mode_yml_path = SerenaAgentMode.get_path(name)
+            is_internal = Path(mode_yml_path).is_relative_to(SERENAS_OWN_MODE_YAMLS_DIR)
+            descriptor = "(internal)" if is_internal else f"(at {mode_yml_path})"
+            name_descr_string = f"{name:<{max_len_name + 4}}{descriptor}"
+            click.echo(name_descr_string)
+
+    @staticmethod
+    @click.command("create", help="Create a new mode or copy an internal one.")
+    @click.option(
+        "--name",
+        type=str,
+        default=None,
+        help="Name for the new mode. If --from-internal is passed may be left empty to create a mode of the same name, which will then override the internal mode.",
+    )
+    @click.option("--from-internal", "from_internal", type=str, default=None, help="Copy from an internal mode.")
+    def create(name: str, from_internal: str) -> None:
+        if not (name or from_internal):
+            raise click.UsageError("Provide at least one of --name or --from-internal.")
+        mode_name = name or from_internal
+        dest = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
+        src = (
+            os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, f"{from_internal}.yml")
+            if from_internal
+            else os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, "mode.template.yml")
+        )
+        if not os.path.exists(src):
+            raise FileNotFoundError(
+                f"Internal mode '{from_internal}' not found in {SERENAS_OWN_MODE_YAMLS_DIR}. Available modes: {SerenaAgentMode.list_registered_mode_names()}"
+            )
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        shutil.copyfile(src, dest)
+        click.echo(f"Created mode '{mode_name}' at {dest}.")
+        _open_in_editor(dest)
+
+    @staticmethod
+    @click.command("edit", help="Edit a custom mode YAML file.")
+    @click.argument("mode_name")
+    def edit(mode_name: str) -> None:
+        path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
+        if not os.path.exists(path):
+            if mode_name in SerenaAgentMode.list_registered_mode_names(include_user_modes=False):
+                click.echo(
+                    f"Mode '{mode_name}' is an internal mode and cannot be edited directly. "
+                    f"Use 'mode create --from-internal {mode_name}' to create a custom mode that overrides it before editing."
+                )
+            else:
+                click.echo(f"Custom mode '{mode_name}' not found. Create it with: mode create --name {mode_name}.")
+            return
+        _open_in_editor(path)
+
+    @staticmethod
+    @click.command("delete", help="Delete a custom mode file.")
+    @click.argument("mode_name")
+    def delete(mode_name: str) -> None:
+        path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
+        if not os.path.exists(path):
+            click.echo(f"Custom mode '{mode_name}' not found.")
+            return
+        os.remove(path)
+        click.echo(f"Deleted custom mode '{mode_name}'.")
+
+
+class ContextCommands(AutoRegisteringGroup):
+    """Group for 'context' subcommands."""
+
+    def __init__(self):
+        super().__init__(
+            name="context", help="Manage Serena contexts. You can run `context <command> --help` for more info on each command."
         )
 
-    log.info("Starting MCP server ...")
+    @staticmethod
+    @click.command("list", help="List available contexts.")
+    def list():
+        context_names = SerenaAgentContext.list_registered_context_names()
+        max_len_name = max(len(name) for name in context_names) if context_names else 20
+        for name in context_names:
+            context_yml_path = SerenaAgentContext.get_path(name)
+            is_internal = Path(context_yml_path).is_relative_to(SERENAS_OWN_CONTEXT_YAMLS_DIR)
+            descriptor = "(internal)" if is_internal else f"(at {context_yml_path})"
+            name_descr_string = f"{name:<{max_len_name + 4}}{descriptor}"
+            click.echo(name_descr_string)
 
-    mcp_server.run(transport=transport)
-
-
-@click.command()
-@click.argument("project", type=click.Path(exists=True), required=False, default=os.getcwd())
-@click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING")
-@click.option(
-    "--only-instructions",
-    is_flag=True,
-    help="If set, only print the initial instructions prompt (without prefix or postfix). The prefix and postfix are small additions that are used to"
-    "make the initial instructions more digestible for an LLM in the context of a system prompt, as opposed to their normal usage within a conversation.",
-)
-def print_system_prompt(project: str, log_level: str = "WARNING", only_instructions: bool = False) -> None:
-    """
-    Print the system prompt (initial instructions with a potential pre/post-fix) for a project.
-    """
-    prefix_to_instructions_prompt = (
-        "You will receive access to Serena's symbolic tools. Below are instructions for using them, take them into account."
+    @staticmethod
+    @click.command("create", help="Create a new context or copy an internal one.")
+    @click.option(
+        "--name",
+        type=str,
+        default=None,
+        help="Name for the new context. If --from-internal is passed may be left empty to create a context of the same name, which will then override the internal context",
     )
-    postfix_to_instructions_prompt = "You begin the conversation by acknowledging that you understood the above instructions on the symbolic tools and are ready to receive a task. You will not need to call the tool on getting initial instructions."
+    @click.option("--from-internal", "from_internal", type=str, default=None, help="Copy from an internal context.")
+    def create(name: str, from_internal: str) -> None:
+        if not (name or from_internal):
+            raise click.UsageError("Provide at least one of --name or --from-internal.")
+        ctx_name = name or from_internal
+        dest = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{ctx_name}.yml")
+        src = (
+            os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, f"{from_internal}.yml")
+            if from_internal
+            else os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, "context.template.yml")
+        )
+        if not os.path.exists(src):
+            raise FileNotFoundError(
+                f"Internal context '{from_internal}' not found in {SERENAS_OWN_CONTEXT_YAMLS_DIR}. Available contexts: {SerenaAgentContext.list_registered_context_names()}"
+            )
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        shutil.copyfile(src, dest)
+        click.echo(f"Created context '{ctx_name}' at {dest}.")
+        _open_in_editor(dest)
 
-    from serena.tools.workflow_tools import InitialInstructionsTool
+    @staticmethod
+    @click.command("edit", help="Edit a custom context YAML file.")
+    @click.argument("context_name")
+    def edit(context_name: str) -> None:
+        path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml")
+        if not os.path.exists(path):
+            if context_name in SerenaAgentContext.list_registered_context_names(include_user_contexts=False):
+                click.echo(
+                    f"Context '{context_name}' is an internal context and cannot be edited directly. "
+                    f"Use 'context create --from-internal {context_name}' to create a custom context that overrides it before editing."
+                )
+            else:
+                click.echo(f"Custom context '{context_name}' not found. Create it with: context create --name {context_name}.")
+            return
+        _open_in_editor(path)
 
-    log_level_int = logging.getLevelNamesMapping()[log_level.upper()]
-    logging.configure(level=log_level_int)
-
-    agent = SerenaAgent(project=os.path.abspath(project), serena_config=SerenaConfig(web_dashboard=False, log_level=log_level_int))
-    initial_instructions_tool = agent.get_tool(InitialInstructionsTool)
-    initial_instructions = initial_instructions_tool.apply()
-    if only_instructions:
-        print(initial_instructions)
-    else:
-        system_prompt = f"{prefix_to_instructions_prompt}\n{initial_instructions}\n{postfix_to_instructions_prompt}"
-        print(system_prompt)
+    @staticmethod
+    @click.command("delete", help="Delete a custom context file.")
+    @click.argument("context_name")
+    def delete(context_name: str) -> None:
+        path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml")
+        if not os.path.exists(path):
+            click.echo(f"Custom context '{context_name}' not found.")
+            return
+        os.remove(path)
+        click.echo(f"Deleted custom context '{context_name}'.")
 
 
-@click.command()
-@click.argument("project", type=click.Path(exists=True), required=False, default=os.getcwd())
-@click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING")
-def index_project(project: str, log_level: str = "INFO") -> None:
-    """
-    Index a project by saving the symbols of files to Serena's language server cache.
+class SerenaConfigCommands(AutoRegisteringGroup):
+    """Group for 'config' subcommands."""
 
-    :param project: the project to index. By default, the current working directory is used.
-    """
-    log_level_int = logging.getLevelNamesMapping()[log_level.upper()]
-    project_instance = Project.load(os.path.abspath(project))
-    print(f"Indexing symbols in project {project}")
-    ls = project_instance.create_language_server(log_level=log_level_int)
-    save_after_n_files = 10
-    with ls.start_server():
-        parsed_files = project_instance.gather_source_files()
-        files_processed = 0
-        pbar = tqdm(parsed_files, disable=False)
-        for relative_file_path in pbar:
-            pbar.set_description(f"Indexing ({os.path.basename(relative_file_path)})")
-            ls.request_document_symbols(relative_file_path, include_body=False)
-            ls.request_document_symbols(relative_file_path, include_body=True)
-            files_processed += 1
-            if files_processed % save_after_n_files == 0:
-                ls.save_cache()
-        ls.save_cache()
-    print(f"Symbols saved to {ls.cache_path}")
+    def __init__(self):
+        super().__init__(name="config", help="Manage Serena configuration.")
+
+    @staticmethod
+    @click.command(
+        "edit", help="Edit serena_config.yml in your default editor. Will create a config file from the template if no config is found."
+    )
+    def edit() -> None:
+        config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, "serena_config.yml")
+        if not os.path.exists(config_path):
+            SerenaConfig.generate_config_file(config_path)
+        _open_in_editor(config_path)
+
+
+class ProjectCommands(AutoRegisteringGroup):
+    """Group for 'project' subcommands."""
+
+    def __init__(self):
+        super().__init__(
+            name="project", help="Manage Serena projects. You can run `project <command> --help` for more info on each command."
+        )
+
+    @staticmethod
+    @click.command("generate-yml", help="Generate a project.yml file.")
+    @click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
+    @click.option("--language", type=str, default=None, help="Programming language; inferred if not specified.")
+    def generate_yml(project_path: str, language: str | None = None) -> None:
+        yml_path = os.path.join(project_path, ProjectConfig.rel_path_to_project_yml())
+        if os.path.exists(yml_path):
+            raise FileExistsError(f"Project file {yml_path} already exists.")
+        lang_inst = None
+        if language:
+            try:
+                lang_inst = cast(Language, Language[language.upper()])
+            except KeyError:
+                all_langs = [l.name.lower() for l in Language.iter_all(include_experimental=True)]
+                raise ValueError(f"Unknown language '{language}'. Supported: {all_langs}")
+        generated_conf = ProjectConfig.autogenerate(project_root=project_path, project_language=lang_inst)
+        print(f"Generated project.yml with language {generated_conf.language.value} at {yml_path}.")
+
+    @staticmethod
+    @click.command("index", help="Index a project by saving symbols to the LSP cache.")
+    @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False)
+    @click.option(
+        "--log-level",
+        type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
+        default="WARNING",
+        help="Log level for indexing.",
+    )
+    def index(project: str, log_level: str = "WARNING") -> None:
+        ProjectCommands._index_project(project, log_level)
+
+    @staticmethod
+    @click.command("index-deprecated", help="Deprecated alias for 'project index'.")
+    @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False)
+    @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING")
+    def index_deprecated(project: str, log_level: str = "WARNING") -> None:
+        click.echo("Deprecated! Use `project index` instead.")
+        ProjectCommands._index_project(project, log_level)
+
+    @staticmethod
+    def _index_project(project: str, log_level: str) -> None:
+        lvl = logging.getLevelNamesMapping()[log_level.upper()]
+        proj = Project.load(os.path.abspath(project))
+        print(f"Indexing symbols in project {project}…")
+        ls = proj.create_language_server(log_level=lvl)
+        with ls.start_server():
+            files = proj.gather_source_files()
+            for i, f in enumerate(tqdm(files, desc="Indexing")):
+                ls.request_document_symbols(f, include_body=False)
+                ls.request_document_symbols(f, include_body=True)
+                if (i + 1) % 10 == 0:
+                    ls.save_cache()
+            ls.save_cache()
+        print(f"Symbols saved to {ls.cache_path}")
+
+
+# Expose groups so we can reference them in pyproject.toml
+mode = ModeCommands()
+context = ContextCommands()
+project = ProjectCommands()
+config = SerenaConfigCommands()
+
+# Expose toplevel commands for the same reason
+top_level = TopLevelCommands()
+start_mcp_server = top_level.start_mcp_server
+index_project = project.index_deprecated
+print_system_prompt = top_level.print_system_prompt
+
+# needed for the help script to work - register all subcommands to the top-level group
+for subgroup in (mode, context, project, config):
+    top_level.add_command(subgroup)
+
+
+def get_help() -> str:
+    """Retrieve the help text for the top-level Serena CLI."""
+    return top_level.get_help(click.Context(top_level, info_name="serena"))

src/serena/config/context_mode.py

@@ -14,7 +14,15 @@ from sensai.util import logging
 from sensai.util.string import ToStringMixin
 
 from serena.config.serena_config import ToolInclusionDefinition
-from serena.constants import CONTEXT_YAMLS_DIR, DEFAULT_CONTEXT, DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, MODE_YAMLS_DIR
+from serena.constants import (
+    DEFAULT_CONTEXT,
+    DEFAULT_MODES,
+    INTERNAL_MODE_YAMLS_DIR,
+    SERENAS_OWN_CONTEXT_YAMLS_DIR,
+    SERENAS_OWN_MODE_YAMLS_DIR,
+    USER_CONTEXT_YAMLS_DIR,
+    USER_MODE_YAMLS_DIR,
+)
 
 if TYPE_CHECKING:
     pass
@@ -61,16 +69,27 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
         name = data.pop("name", Path(yaml_path).stem)
         return cls(name=name, **data)
 
+    @classmethod
+    def get_path(cls, name: str) -> str:
+        """Get the path to the YAML file for a mode."""
+        fname = f"{name}.yml"
+        custom_mode_path = os.path.join(USER_MODE_YAMLS_DIR, fname)
+        if os.path.exists(custom_mode_path):
+            return custom_mode_path
+
+        own_yaml_path = os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, fname)
+        if not os.path.exists(own_yaml_path):
+            raise FileNotFoundError(
+                f"Mode {name} not found in {USER_MODE_YAMLS_DIR} or in {SERENAS_OWN_MODE_YAMLS_DIR}."
+                f"Available modes:\n{cls.list_registered_mode_names()}"
+            )
+        return own_yaml_path
+
     @classmethod
     def from_name(cls, name: str) -> Self:
         """Load a registered Serena mode."""
-        yaml_path = os.path.join(MODE_YAMLS_DIR, f"{name}.yml")
-        if not os.path.exists(yaml_path):
-            raise FileNotFoundError(
-                f"Mode {name} not found in {MODE_YAMLS_DIR}. You can load custom modes by using from_yaml() instead. "
-                f"Available modes: {cls.list_registered_mode_names()}"
-            )
-        return cls.from_yaml(yaml_path)
+        mode_path = cls.get_path(name)
+        return cls.from_yaml(mode_path)
 
     @classmethod
     def from_name_internal(cls, name: str) -> Self:
@@ -81,9 +100,17 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
         return cls.from_yaml(yaml_path)
 
     @classmethod
-    def list_registered_mode_names(cls) -> list[str]:
+    def list_registered_mode_names(cls, include_user_modes: bool = True) -> list[str]:
         """Names of all registered modes (from the corresponding YAML files in the serena repo)."""
-        return sorted([f.stem for f in Path(MODE_YAMLS_DIR).glob("*.yml")])
+        modes = [f.stem for f in Path(SERENAS_OWN_MODE_YAMLS_DIR).glob("*.yml") if f.name != "mode.template.yml"]
+        if include_user_modes:
+            modes += cls.list_custom_mode_names()
+        return sorted(set(modes))
+
+    @classmethod
+    def list_custom_mode_names(cls) -> list[str]:
+        """Names of all custom modes defined by the user."""
+        return [f.stem for f in Path(USER_MODE_YAMLS_DIR).glob("*.yml")]
 
     @classmethod
     def load_default_modes(cls) -> list[Self]:
@@ -92,10 +119,9 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
 
     @classmethod
     def load(cls, name_or_path: str | Path) -> Self:
-        try:
-            return cls.from_name(str(name_or_path))
-        except FileNotFoundError:
+        if str(name_or_path).endswith(".yml"):
             return cls.from_yaml(name_or_path)
+        return cls.from_name(str(name_or_path))
 
 
 @dataclass(kw_only=True)
@@ -132,33 +158,45 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
         return cls(name=name, **data)
 
     @classmethod
-    def from_name(cls, name: str) -> Self:
-        """Load a registered Serena context."""
-        yaml_path = os.path.join(CONTEXT_YAMLS_DIR, f"{name}.yml")
-        if not os.path.exists(yaml_path):
+    def get_path(cls, name: str) -> str:
+        """Get the path to the YAML file for a context."""
+        fname = f"{name}.yml"
+        custom_context_path = os.path.join(USER_CONTEXT_YAMLS_DIR, fname)
+        if os.path.exists(custom_context_path):
+            return custom_context_path
+
+        own_yaml_path = os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, fname)
+        if not os.path.exists(own_yaml_path):
             raise FileNotFoundError(
-                f"Context {Path(yaml_path).stem} not found in {CONTEXT_YAMLS_DIR}. You can load a custom context by using from_yaml() instead.\n"
+                f"Context {name} not found in {USER_CONTEXT_YAMLS_DIR} or in {SERENAS_OWN_CONTEXT_YAMLS_DIR}."
                 f"Available contexts:\n{cls.list_registered_context_names()}"
             )
-        return cls.from_yaml(yaml_path)
+        return own_yaml_path
+
+    @classmethod
+    def from_name(cls, name: str) -> Self:
+        """Load a registered Serena context."""
+        context_path = cls.get_path(name)
+        return cls.from_yaml(context_path)
 
     @classmethod
     def load(cls, name_or_path: str | Path) -> Self:
-        try:
-            return cls.from_name(str(name_or_path))
-        except FileNotFoundError:
-            try:
-                return cls.from_yaml(name_or_path)
-            except FileNotFoundError as e:
-                raise FileNotFoundError(
-                    f"Context {name_or_path} not found in {CONTEXT_YAMLS_DIR}. You can load a custom context by using from_yaml() instead.\n"
-                    f"Available contexts:\n{cls.list_registered_context_names()}"
-                ) from e
+        if str(name_or_path).endswith(".yml"):
+            return cls.from_yaml(name_or_path)
+        return cls.from_name(str(name_or_path))
 
     @classmethod
-    def list_registered_context_names(cls) -> list[str]:
+    def list_registered_context_names(cls, include_user_contexts: bool = True) -> list[str]:
         """Names of all registered contexts (from the corresponding YAML files in the serena repo)."""
-        return sorted([f.stem for f in Path(CONTEXT_YAMLS_DIR).glob("*.yml")])
+        contexts = [f.stem for f in Path(SERENAS_OWN_CONTEXT_YAMLS_DIR).glob("*.yml")]
+        if include_user_contexts:
+            contexts += cls.list_custom_context_names()
+        return sorted(set(contexts))
+
+    @classmethod
+    def list_custom_context_names(cls) -> list[str]:
+        """Names of all custom contexts defined by the user."""
+        return [f.stem for f in Path(USER_CONTEXT_YAMLS_DIR).glob("*.yml")]
 
     @classmethod
     def load_default(cls) -> Self:

src/serena/config/serena_config.py

@@ -21,6 +21,7 @@ from serena.constants import (
     PROJECT_TEMPLATE_FILE,
     REPO_ROOT,
     SELENA_CONFIG_TEMPLATE_FILE,
+    SERENA_MANAGED_DIR_IN_HOME,
     SERENA_MANAGED_DIR_NAME,
 )
 from serena.util.general import load_yaml, save_yaml
@@ -113,7 +114,7 @@ class SerenaConfigError(Exception):
     pass
 
 
-def get_serena_managed_dir(project_root: str | Path) -> str:
+def get_serena_managed_in_project_dir(project_root: str | Path) -> str:
     return os.path.join(project_root, SERENA_MANAGED_DIR_NAME)
 
 
@@ -146,13 +147,16 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
         return ["project_name"]
 
     @classmethod
-    def autogenerate(cls, project_root: str | Path, project_name: str | None = None, save_to_disk: bool = True) -> Self:
+    def autogenerate(
+        cls, project_root: str | Path, project_name: str | None = None, project_language: Language | None = None, save_to_disk: bool = True
+    ) -> Self:
         """
         Autogenerate a project configuration for a given project root.
 
         :param project_root: the path to the project root
         :param project_name: the name of the project; if None, the name of the project will be the name of the directory
             containing the project
+        :param project_language: the programming language of the project; if None, it will be determined automatically
         :param save_to_disk: whether to save the project configuration to disk
         :return: the project configuration
         """
@@ -160,20 +164,23 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
         if not project_root.exists():
             raise FileNotFoundError(f"Project root not found: {project_root}")
         project_name = project_name or project_root.name
-        language_composition = determine_programming_language_composition(str(project_root))
-        if len(language_composition) == 0:
-            raise ValueError(
-                f"No source files found in {project_root}\n\n"
-                f"To use Serena with this project, you need to either:\n"
-                f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP)\n"
-                f"2. Create a project configuration file manually at:\n"
-                f"   {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n"
-                f"Example project.yml:\n"
-                f"  project_name: {project_name}\n"
-                f"  language: python  # or typescript, java, csharp, rust, go, ruby, cpp, php\n"
-            )
-        # find the language with the highest percentage
-        dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
+        if project_language is None:
+            language_composition = determine_programming_language_composition(str(project_root))
+            if len(language_composition) == 0:
+                raise ValueError(
+                    f"No source files found in {project_root}\n\n"
+                    f"To use Serena with this project, you need to either:\n"
+                    f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP)\n"
+                    f"2. Create a project configuration file manually at:\n"
+                    f"   {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n"
+                    f"Example project.yml:\n"
+                    f"  project_name: {project_name}\n"
+                    f"  language: python  # or typescript, java, csharp, rust, go, ruby, cpp, php\n"
+                )
+            # find the language with the highest percentage
+            dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
+        else:
+            dominant_language = project_language.value
         config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True)
         config_with_comments["project_name"] = project_name
         config_with_comments["language"] = dominant_language
@@ -275,7 +282,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         return ["config_file_path"]
 
     @classmethod
-    def _generate_config_file(cls, config_file_path: str) -> None:
+    def generate_config_file(cls, config_file_path: str) -> None:
         """
         Generates a Serena configuration file at the specified path from the template file.
 
@@ -293,7 +300,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         if is_running_in_docker():
             return os.path.join(REPO_ROOT, cls.CONFIG_FILE_DOCKER)
         else:
-            config_path = str(Path.home() / SERENA_MANAGED_DIR_NAME / cls.CONFIG_FILE)
+            config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, cls.CONFIG_FILE)
 
             # if the config file does not exist, check if we can migrate it from the old location
             if not os.path.exists(config_path):
@@ -319,7 +326,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
             if not generate_if_missing:
                 raise FileNotFoundError(f"Serena configuration file not found: {config_file_path}")
             log.info(f"Serena configuration file not found at {config_file_path}, autogenerating...")
-            cls._generate_config_file(config_file_path)
+            cls.generate_config_file(config_file_path)
 
         # load the configuration
         log.info(f"Loading Serena configuration from {config_file_path}")

src/serena/constants.py

@@ -3,15 +3,26 @@ from pathlib import Path
 _repo_root_path = Path(__file__).parent.parent.parent.resolve()
 _serena_pkg_path = Path(__file__).parent.resolve()
 
+SERENA_MANAGED_DIR_NAME = ".serena"
+_serena_in_home_managed_dir = Path.home() / ".serena"
+
+SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir)
+
 REPO_ROOT = str(_repo_root_path)
 PROMPT_TEMPLATES_DIR = str(_serena_pkg_path / "resources" / "config" / "prompt_templates")
-CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts")
-MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "modes")
+SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts")
+"""The contexts that are shipped with the Serena package, i.e. the default contexts."""
+USER_CONTEXT_YAMLS_DIR = str(_serena_in_home_managed_dir / "contexts")
+"""Contexts defined by the user. If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR, the user context will override the default one."""
+SERENAS_OWN_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "modes")
+"""The modes that are shipped with the Serena package, i.e. the default modes."""
+USER_MODE_YAMLS_DIR = str(_serena_in_home_managed_dir / "modes")
+"""Modes defined by the user. If a name of a mode matches a name of a mode in SERENAS_OWN_MODE_YAMLS_DIR, the user mode will override the default one."""
 INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "internal_modes")
+"""Internal modes, never overridden by user modes."""
 SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard")
 SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
 
-SERENA_MANAGED_DIR_NAME = ".serena"
 
 DEFAULT_ENCODING = "utf-8"
 DEFAULT_CONTEXT = "desktop-app"

src/serena/resources/config/contexts/context.template.yml

@@ -0,0 +1,4 @@
+# See Serena's documentation for more details on concept of contexts.
+description: Description of the context, not used in the code.
+prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context.
+excluded_tools: []

src/serena/resources/config/modes/mode.template.yml

@@ -0,0 +1,4 @@
+# See Serena's documentation for more details on concept of modes.
+description: Description of the mode, not used in the code.
+prompt: Prompt that will form part of the message sent to the model when activating this mode.
+excluded_tools: []