diff --git a/.gitignore b/.gitignore index 9fdcc92..ce9c54f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,64 @@ -# +# macOS specific files +.DS_Store +.AppleDouble +.LSOverride +._* +.Spotlight-V100 +.Trashes +Icon +.fseventsd +.DocumentRevisions-V100 +.TemporaryItems +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Windows specific files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.stackdump +[Dd]esktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux specific files +*~ +.fuse_hidden* +.directory +.Trash-* +.nfs* + +# IDE/Text Editors +# VS Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace +.history/ + +# JetBrains IDEs (beyond .idea/) +*.iml +*.ipr +*.iws +out/ +.idea_modules/ + +# Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# Project specific ignore .idea temp diff --git a/pyproject.toml b/pyproject.toml index 16ebc86..5ef68fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "psutil>=7.0.0", "agno>=1.2.15", "docstring_parser>=0.16", + "joblib>=1.5.1", ] [project.scripts] @@ -257,4 +258,4 @@ markers = [ "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", -] \ No newline at end of file +] diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 6ca98b9..fab22e5 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -136,11 +136,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - from multilspy.language_servers.jedi_language_server.jedi_server import ( - JediServer, - ) + # from multilspy.language_servers.jedi_language_server.jedi_server import ( + # JediServer, + # ) - return JediServer(config, logger, repository_root_path) + # return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -1177,42 +1177,25 @@ class LanguageServer: async def request_parsed_files(self) -> list[str]: - """ - Retrieves relative paths of all files analyzed by the Language Server. - - This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" if not self.server_started: self.logger.log( "request_parsed_files called before Language Server started", logging.ERROR, ) raise MultilspyException("Language Server not started") - # TODO: this worked in jedi, but pyright and basedpyright return nothing... - # I don't know why - # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - # symbols = await self.server.send.workspace_symbol(params) or [] - - # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which - # seems to work in all these language servers - # walk through all children recursively, find all symbols of type Module and collect their relative paths - roots = await self.request_full_symbol_tree() - paths = [] - def collect_module_files(symbol): - if symbol["kind"] == multilspy_types.SymbolKind.File: - assert "location" in symbol - paths.append(symbol["location"]["relativePath"]) - - elif symbol["kind"] == multilspy_types.SymbolKind.Package: - for child in symbol["children"]: - collect_module_files(child) - - for root in roots: - collect_module_files(root) - - return paths - + rel_file_paths = [] + for root, dirs, files in os.walk(self.repository_root_path): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + for file in files: + rel_file_path = os.path.join(root, file) + if not self.is_ignored_path(rel_file_path): + rel_file_paths.append(rel_file_path) + return rel_file_paths async def search_files_for_pattern( self, @@ -1926,9 +1909,7 @@ class SyncLanguageServer: return self.language_server.retrieve_symbol_body(symbol) def request_parsed_files(self) -> list[str]: - """This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" assert self.loop result = asyncio.run_coroutine_threadsafe( self.language_server.request_parsed_files(), self.loop diff --git a/src/serena/agno.py b/src/serena/agno.py index 3e3fec4..f6684ec 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -70,20 +70,32 @@ class SerenaAgnoAgentProvider: load_dotenv() parser = argparse.ArgumentParser(description="Serena coding assistant") - parser.add_argument( + + # Create a mutually exclusive group + group = parser.add_mutually_exclusive_group() + + # Add arguments to the group, both pointing to the same destination + group.add_argument( + "--project-file", + required=False, + help="Path to the project (or project.yml file).", + ) + group.add_argument( "--project", required=False, help="Path to the project (or project.yml file).", ) args = parser.parse_args() - if args.project_file: - project_file = Path(args.project_file).resolve() + args_project_file = args.project or args.project_file + + if args_project_file: + project_file = Path(args_project_file).resolve() # If project file path is relative, make it absolute by joining with project root if not project_file.is_absolute(): # Get the project root directory (parent of scripts directory) project_root = Path(serena_root_path()) - project_file = project_root / args.project_file + project_file = project_root / args_project_file # Ensure the path is normalized and absolute project_file = str(project_file.resolve()) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 171e209..3178d55 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -20,7 +20,8 @@ log = logging.getLogger(__name__) @dataclass class SymbolLocation: """ - Represents the (start) location of a symbol identifier + Represents the location of a symbol, including the line where the identifier + is defined and the end line of the symbol's body """ relative_path: str | None @@ -37,6 +38,16 @@ class SymbolLocation: the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.); may be None for some types of symbols (e.g. SymbolKind.File) """ + end_line: int | None = None + """ + the line in which the symbol's body ends. For methods that search based on SymbolLocation, + the end line is not needed, as it is unused by the language server in the search. + However, the end_line will typically be included in the results of search methods, + and thus be provided to the user in the response to such requests. + This is useful for the LLMs, especially the less intelligent ones, as they often fail + to perform symbolic operations and when falling back to line-editing tools, will just make up + an end line. + """ def __post_init__(self) -> None: if self.relative_path is not None: @@ -118,10 +129,7 @@ class Symbol(ToStringMixin): @property def location(self) -> SymbolLocation: - """ - :return: the start location of the actual symbol identifier - """ - return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column) + return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line) @property def body_start_position(self) -> Position | None: @@ -147,12 +155,21 @@ class Symbol(ToStringMixin): @property def line(self) -> int | None: + """The line in which the symbol identifier is defined (start line).""" if "selectionRange" in self.symbol_root: return self.symbol_root["selectionRange"]["start"]["line"] else: # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File) return None + @property + def end_line(self) -> int | None: + """The end line of the symbol body, also contained in the `body_end_position`.""" + body_end_position = self.body_end_position + if body_end_position is not None: + return body_end_position["line"] + return None + @property def column(self) -> int | None: if "selectionRange" in self.symbol_root: @@ -365,7 +382,8 @@ class SymbolManager: """ Find all symbols that reference the symbol at the given location. - :param symbol_location: the location of the symbol for which to find references + :param symbol_location: the location of the symbol for which to find references. + Does not need to include an end_line, as it is unused in the search. :param include_body: whether to include the body of all symbols in the result. Note: you can filter out the bodies of the children if you set include_children_body=False in the to_dict method. @@ -420,7 +438,7 @@ class SymbolManager: """ Replace the body of the symbol at the given location with the given body - :param location: the location of the symbol to replace + :param location: the location of the symbol to replace. :param body: the new body """ # make sure body always ends with at least one newline diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 1bae75a..6cab02a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,7 +3,9 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from typing import Any +from joblib import Parallel, delayed from pathspec import PathSpec from pathspec.patterns.gitwildmatch import GitWildMatchPattern @@ -103,7 +105,8 @@ def search_text( content: The text content to search. May be None if source_file_path is provided. source_file_path: Optional path to the source file. If content is None, this has to be passed and the file will be read. - allow_multiline_match: Whether to search across multiple lines + allow_multiline_match: Whether to search across multiple lines. Currently, the default + option (False) is very inefficient, so it is recommended to set this to True. context_lines_before: Number of context lines to include before matches context_lines_after: Number of context lines to include after matches is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") @@ -188,6 +191,8 @@ def search_text( matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path)) else: + # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer, + # it is not urgent to fix, but should be either improved or the option should be removed. # Search line by line for i, line in enumerate(lines): line_num = i + 1 @@ -242,10 +247,11 @@ def search_files( :param paths_exclude_glob: Optional glob pattern to exclude files from the list :return: List of MatchedConsecutiveLines objects """ - matches = [] + # Pre-filter paths (done sequentially to avoid overhead) include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None - skipped_file_error_tuples: list[tuple[str, str]] = [] + + filtered_paths = [] for path in file_paths: if include_spec and not include_spec.match_file(path): log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") @@ -253,26 +259,47 @@ def search_files( if exclude_spec and exclude_spec.match_file(path): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + filtered_paths.append(path) + + log.info(f"Processing {len(filtered_paths)} files.") + + def process_single_file(path: str) -> dict[str, Any]: + """Process a single file - this function will be parallelized.""" try: file_content = file_reader(path) + search_results = search_text( + pattern, + content=file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + ) + if len(search_results) > 0: + log.debug(f"Found {len(search_results)} matches in {path}") + return {"path": path, "results": search_results, "error": None} except Exception as e: - skipped_file_error_tuples.append((path, str(e))) - continue + log.debug(f"Error processing {path}: {e}") + return {"path": path, "results": [], "error": str(e)} + + # Execute in parallel using joblib + results = Parallel( + n_jobs=-1, + backend="threading", + )(delayed(process_single_file)(path) for path in filtered_paths) + + # Collect results and errors + matches = [] + skipped_file_error_tuples = [] + + for result in results: + if result["error"]: + skipped_file_error_tuples.append((result["path"], result["error"])) + else: + matches.extend(result["results"]) - search_results = search_text( - pattern, - file_content, - source_file_path=path, - allow_multiline_match=True, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - ) - if len(search_results) > 0: - log.debug(f"Found {len(search_results)} matches in {path}") - matches.extend(search_results) if skipped_file_error_tuples: - log.debug( - f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}" - ) + log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}") + log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") return matches diff --git a/src/serena/util/shell.py b/src/serena/util/shell.py index 3d9e66e..d940384 100644 --- a/src/serena/util/shell.py +++ b/src/serena/util/shell.py @@ -1,4 +1,5 @@ import os +import platform import subprocess from pydantic import BaseModel @@ -25,7 +26,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr: process = subprocess.Popen( command, - shell=True, + shell=platform.system() != "Windows", stdout=subprocess.PIPE, stderr=subprocess.PIPE if capture_stderr else None, text=True, diff --git a/uv.lock b/uv.lock index 5390942..b9d538e 100644 --- a/uv.lock +++ b/uv.lock @@ -455,6 +455,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" }, ] +[[package]] +name = "joblib" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -967,6 +976,7 @@ dependencies = [ { name = "fastapi" }, { name = "fastmcp" }, { name = "jinja2" }, + { name = "joblib" }, { name = "mcp" }, { name = "overrides" }, { name = "pathspec" }, @@ -1016,6 +1026,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, + { name = "joblib", specifier = ">=1.5.1" }, { name = "mcp", specifier = ">=1.5.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" }, { name = "overrides", specifier = ">=7.7.0,<8" },