From c53a84487010bd7de78ac4a8007fe136182b74b3 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sat, 24 May 2025 21:26:18 +0200 Subject: [PATCH 01/14] Minor, don't exclude tools in one-shot mode --- config/modes/one-shot.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/config/modes/one-shot.yml b/config/modes/one-shot.yml index 1d0f061..9b5e289 100644 --- a/config/modes/one-shot.yml +++ b/config/modes/one-shot.yml @@ -12,7 +12,4 @@ prompt: | It may be that you have not received a task yet. In this case, wait for the user to provide a task, this will be the only time you should wait for user interaction. -excluded_tools: - - get_current_config - - activate_project - - switch_modes +excluded_tools: [] From 45f27ec90ab756c07b6a0c682b618aba9696d83a Mon Sep 17 00:00:00 2001 From: Anastasios Iliou Date: Tue, 27 May 2025 11:16:23 +0300 Subject: [PATCH 02/14] added mutually exclusive group, corrected parsing --- src/serena/agno.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/serena/agno.py b/src/serena/agno.py index 05c4ffd..19ad081 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -70,20 +70,32 @@ class SerenaAgnoAgentProvider: load_dotenv() parser = argparse.ArgumentParser(description="Serena coding assistant") - parser.add_argument( + + # Create a mutually exclusive group + group = parser.add_mutually_exclusive_group() + + # Add arguments to the group, both pointing to the same destination + group.add_argument( + "--project-file", + required=False, + help="Path to the project (or project.yml file).", + ) + group.add_argument( "--project", required=False, help="Path to the project (or project.yml file).", ) args = parser.parse_args() - if args.project_file: - project_file = Path(args.project_file).resolve() + args_project_file = args.project or args.project_file + + if args_project_file: + project_file = Path(args_project_file).resolve() # If project file path is relative, make it absolute by joining with project root if not project_file.is_absolute(): # Get the project root directory (parent of scripts directory) project_root = Path(serena_root_path()) - project_file = project_root / args.project_file + project_file = project_root / args_project_file # Ensure the path is normalized and absolute project_file = str(project_file.resolve()) From 02174d789abafb5ea14ba0bbc18d67dd675d3734 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 May 2025 15:24:46 +0200 Subject: [PATCH 03/14] Formatting --- src/serena/agent.py | 4 ++-- src/serena/agno.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 527bfc2..0a82006 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -324,7 +324,7 @@ class SerenaAgent: self._context = context self._modes = modes log.info(f"Loaded tools ({len(self._all_tools)}): {', '.join([tool.get_name() for tool in self._all_tools.values()])}") - + self._active_tools: dict[type[Tool], Tool] = {} self._update_active_tools() @@ -416,7 +416,7 @@ class SerenaAgent: log.info(f"Activating {project_config}") self.project_config = project_config self._update_active_tools() - + # start the language server self.reset_language_server() assert self.language_server is not None diff --git a/src/serena/agno.py b/src/serena/agno.py index 19ad081..a103129 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -76,7 +76,7 @@ class SerenaAgnoAgentProvider: # Add arguments to the group, both pointing to the same destination group.add_argument( - "--project-file", + "--project-file", required=False, help="Path to the project (or project.yml file).", ) @@ -87,7 +87,7 @@ class SerenaAgnoAgentProvider: ) args = parser.parse_args() - args_project_file = args.project or args.project_file + args_project_file = args.project or args.project_file if args_project_file: project_file = Path(args_project_file).resolve() From 4d5ab9a88078a645dddaa012c39bec2725b6ef48 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 May 2025 15:23:54 +0200 Subject: [PATCH 04/14] execute_shell_command: Use shell=False on Windows --- src/serena/util/shell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/serena/util/shell.py b/src/serena/util/shell.py index 3d9e66e..d940384 100644 --- a/src/serena/util/shell.py +++ b/src/serena/util/shell.py @@ -1,4 +1,5 @@ import os +import platform import subprocess from pydantic import BaseModel @@ -25,7 +26,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr: process = subprocess.Popen( command, - shell=True, + shell=platform.system() != "Windows", stdout=subprocess.PIPE, stderr=subprocess.PIPE if capture_stderr else None, text=True, From 6c121836bc10ad4fbbb50a47241d65155b34be6f Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 27 May 2025 17:30:55 +0200 Subject: [PATCH 05/14] Add end_line to serena's Symbol representation Needed for LLMs to not attempt guessing end lines when reading symbols with non-symbolic tools --- src/serena/symbol.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 171e209..3b939b8 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -20,7 +20,8 @@ log = logging.getLogger(__name__) @dataclass class SymbolLocation: """ - Represents the (start) location of a symbol identifier + Represents the location of a symbol, including the line where the identifier + is defined and the end line of the symbol's body """ relative_path: str | None @@ -37,6 +38,10 @@ class SymbolLocation: the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.); may be None for some types of symbols (e.g. SymbolKind.File) """ + end_line: int | None + """ + the line in which the symbol's body ends, may be None (e.g., if line is None) + """ def __post_init__(self) -> None: if self.relative_path is not None: @@ -121,7 +126,7 @@ class Symbol(ToStringMixin): """ :return: the start location of the actual symbol identifier """ - return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column) + return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line) @property def body_start_position(self) -> Position | None: @@ -147,11 +152,20 @@ class Symbol(ToStringMixin): @property def line(self) -> int | None: + """The line in which the symbol identifier is defined (start line).""" if "selectionRange" in self.symbol_root: return self.symbol_root["selectionRange"]["start"]["line"] else: # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File) return None + + @property + def end_line(self) -> int | None: + """The end line of the symbol body, also contained in the `body_end_position`.""" + body_end_position = self.body_end_position + if body_end_position is not None: + return body_end_position["line"] + return None @property def column(self) -> int | None: From 852a36ce1820afdf8a665639006b68b29802ccf5 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Tue, 27 May 2025 17:40:16 +0200 Subject: [PATCH 06/14] Docstring [ci skip] --- src/serena/symbol.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 3b939b8..8f3ab47 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -123,9 +123,6 @@ class Symbol(ToStringMixin): @property def location(self) -> SymbolLocation: - """ - :return: the start location of the actual symbol identifier - """ return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line) @property From 225688c0142d420c3ad08691da887c8ab730a15a Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 11:44:11 +0200 Subject: [PATCH 07/14] Added joblib to dependencies --- pyproject.toml | 3 ++- uv.lock | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 81ca857..02fe562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "psutil>=7.0.0", "agno>=1.2.15", "docstring_parser>=0.16", + "joblib>=1.5.1", ] [project.scripts] @@ -256,4 +257,4 @@ markers = [ "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", -] \ No newline at end of file +] diff --git a/uv.lock b/uv.lock index eef6518..4314c74 100644 --- a/uv.lock +++ b/uv.lock @@ -455,6 +455,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" }, ] +[[package]] +name = "joblib" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -937,6 +946,7 @@ dependencies = [ { name = "dotenv" }, { name = "fastmcp" }, { name = "jinja2" }, + { name = "joblib" }, { name = "mcp" }, { name = "overrides" }, { name = "pathspec" }, @@ -986,6 +996,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, + { name = "joblib", specifier = ">=1.5.1" }, { name = "mcp", specifier = ">=1.5.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" }, { name = "overrides", specifier = ">=7.7.0,<8" }, From 3cbd3559ece786c5aea676272456d37c4c53b4ec Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 11:45:36 +0200 Subject: [PATCH 08/14] Parallelize file processinging in search_files (WIP) --- src/serena/text_utils.py | 65 +++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 1bae75a..41ec0dc 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,6 +3,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from joblib import Parallel, delayed from pathspec import PathSpec from pathspec.patterns.gitwildmatch import GitWildMatchPattern @@ -242,10 +243,11 @@ def search_files( :param paths_exclude_glob: Optional glob pattern to exclude files from the list :return: List of MatchedConsecutiveLines objects """ - matches = [] + # Pre-filter paths (done sequentially to avoid overhead) include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None - skipped_file_error_tuples: list[tuple[str, str]] = [] + + filtered_paths = [] for path in file_paths: if include_spec and not include_spec.match_file(path): log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") @@ -253,26 +255,47 @@ def search_files( if exclude_spec and exclude_spec.match_file(path): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + filtered_paths.append(path) + + log.info(f"Processing {len(filtered_paths)} files.") + + def process_single_file(path: str): + """Process a single file - this function will be parallelized.""" try: file_content = file_reader(path) + search_results = search_text( + pattern, + content=file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + ) + if len(search_results) > 0: + log.debug(f"Found {len(search_results)} matches in {path}") + return {'path': path, 'results': search_results, 'error': None} except Exception as e: - skipped_file_error_tuples.append((path, str(e))) - continue - - search_results = search_text( - pattern, - file_content, - source_file_path=path, - allow_multiline_match=True, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - ) - if len(search_results) > 0: - log.debug(f"Found {len(search_results)} matches in {path}") - matches.extend(search_results) + log.debug(f"Error processing {path}: {e}") + return {'path': path, 'results': [], 'error': str(e)} + + # Execute in parallel using joblib + results = Parallel( + n_jobs=-1, + backend="threading", + )(delayed(process_single_file)(path) for path in filtered_paths) + + # Collect results and errors + matches = [] + skipped_file_error_tuples = [] + + for result in results: + if result['error']: + skipped_file_error_tuples.append((result['path'], result['error'])) + else: + matches.extend(result['results']) + if skipped_file_error_tuples: - log.debug( - f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}" - ) - - return matches + log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}") + + log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") + return matches \ No newline at end of file From 269d4525c2c8c1e57a46e1fdd7b31d174075bbaa Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 15:15:41 +0200 Subject: [PATCH 09/14] Gitignore --- .gitignore | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fc86216..58bc84a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,64 @@ -# +# macOS specific files +.DS_Store +.AppleDouble +.LSOverride +._* +.Spotlight-V100 +.Trashes +Icon +.fseventsd +.DocumentRevisions-V100 +.TemporaryItems +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Windows specific files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.stackdump +[Dd]esktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux specific files +*~ +.fuse_hidden* +.directory +.Trash-* +.nfs* + +# IDE/Text Editors +# VS Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace +.history/ + +# JetBrains IDEs (beyond .idea/) +*.iml +*.ipr +*.iws +out/ +.idea_modules/ + +# Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# Project specific ignore .idea temp From 0ce2f0d4d621839cde6bc248489370db1013020c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 17:13:03 +0200 Subject: [PATCH 10/14] Hotfix: make end_line optional in SymbolLocation We use SymbolLocation not only within responses but also within requests, where the end line is not known --- src/serena/symbol.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 8f3ab47..3178d55 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -38,9 +38,15 @@ class SymbolLocation: the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.); may be None for some types of symbols (e.g. SymbolKind.File) """ - end_line: int | None + end_line: int | None = None """ - the line in which the symbol's body ends, may be None (e.g., if line is None) + the line in which the symbol's body ends. For methods that search based on SymbolLocation, + the end line is not needed, as it is unused by the language server in the search. + However, the end_line will typically be included in the results of search methods, + and thus be provided to the user in the response to such requests. + This is useful for the LLMs, especially the less intelligent ones, as they often fail + to perform symbolic operations and when falling back to line-editing tools, will just make up + an end line. """ def __post_init__(self) -> None: @@ -155,7 +161,7 @@ class Symbol(ToStringMixin): else: # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File) return None - + @property def end_line(self) -> int | None: """The end line of the symbol body, also contained in the `body_end_position`.""" @@ -376,7 +382,8 @@ class SymbolManager: """ Find all symbols that reference the symbol at the given location. - :param symbol_location: the location of the symbol for which to find references + :param symbol_location: the location of the symbol for which to find references. + Does not need to include an end_line, as it is unused in the search. :param include_body: whether to include the body of all symbols in the result. Note: you can filter out the bodies of the children if you set include_children_body=False in the to_dict method. @@ -431,7 +438,7 @@ class SymbolManager: """ Replace the body of the symbol at the given location with the given body - :param location: the location of the symbol to replace + :param location: the location of the symbol to replace. :param body: the new body """ # make sure body always ends with at least one newline From e4453b73e2e97e3c741924fddf6a831d3c778193 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 17:51:18 +0200 Subject: [PATCH 11/14] LS: massively sped up request_parsed_files --- src/multilspy/language_server.py | 44 ++++++++++++-------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 6ca98b9..8ec22d0 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -136,11 +136,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - from multilspy.language_servers.jedi_language_server.jedi_server import ( - JediServer, - ) + # from multilspy.language_servers.jedi_language_server.jedi_server import ( + # JediServer, + # ) - return JediServer(config, logger, repository_root_path) + # return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -1189,30 +1189,18 @@ class LanguageServer: logging.ERROR, ) raise MultilspyException("Language Server not started") - # TODO: this worked in jedi, but pyright and basedpyright return nothing... - # I don't know why - # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - # symbols = await self.server.send.workspace_symbol(params) or [] - - # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which - # seems to work in all these language servers - # walk through all children recursively, find all symbols of type Module and collect their relative paths - roots = await self.request_full_symbol_tree() - paths = [] - def collect_module_files(symbol): - if symbol["kind"] == multilspy_types.SymbolKind.File: - assert "location" in symbol - paths.append(symbol["location"]["relativePath"]) - - elif symbol["kind"] == multilspy_types.SymbolKind.Package: - for child in symbol["children"]: - collect_module_files(child) - - for root in roots: - collect_module_files(root) - - return paths - + rel_file_paths = [] + for root, dirs, files in os.walk(self.repository_root_path): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + for file in files: + rel_file_path = os.path.join(root, file) + if not self.is_ignored_path(rel_file_path): + rel_file_paths.append(rel_file_path) + return rel_file_paths async def search_files_for_pattern( self, From ad753d308c751c1c87b9f8c0dc86fb0683e7d05c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 17:53:11 +0200 Subject: [PATCH 12/14] Massively sped up search_files by parallelizing reading of files Used multithreading for executing file_reader --- src/serena/symbol.py | 2 +- src/serena/text_utils.py | 35 +++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 8f3ab47..f426bfc 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -155,7 +155,7 @@ class Symbol(ToStringMixin): else: # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File) return None - + @property def end_line(self) -> int | None: """The end line of the symbol body, also contained in the `body_end_position`.""" diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 41ec0dc..aa0b2b7 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,8 +3,8 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum -from joblib import Parallel, delayed +from joblib import Parallel, delayed from pathspec import PathSpec from pathspec.patterns.gitwildmatch import GitWildMatchPattern @@ -104,7 +104,8 @@ def search_text( content: The text content to search. May be None if source_file_path is provided. source_file_path: Optional path to the source file. If content is None, this has to be passed and the file will be read. - allow_multiline_match: Whether to search across multiple lines + allow_multiline_match: Whether to search across multiple lines. Currently, the default + option (False) is very inefficient, so it is recommended to set this to True. context_lines_before: Number of context lines to include before matches context_lines_after: Number of context lines to include after matches is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") @@ -189,6 +190,8 @@ def search_text( matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path)) else: + # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer, + # it is not urgent to fix, but should be either improved or the option should be removed. # Search line by line for i, line in enumerate(lines): line_num = i + 1 @@ -246,7 +249,7 @@ def search_files( # Pre-filter paths (done sequentially to avoid overhead) include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None - + filtered_paths = [] for path in file_paths: if include_spec and not include_spec.match_file(path): @@ -256,9 +259,9 @@ def search_files( log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue filtered_paths.append(path) - + log.info(f"Processing {len(filtered_paths)} files.") - + def process_single_file(path: str): """Process a single file - this function will be parallelized.""" try: @@ -273,29 +276,29 @@ def search_files( ) if len(search_results) > 0: log.debug(f"Found {len(search_results)} matches in {path}") - return {'path': path, 'results': search_results, 'error': None} + return {"path": path, "results": search_results, "error": None} except Exception as e: log.debug(f"Error processing {path}: {e}") - return {'path': path, 'results': [], 'error': str(e)} - + return {"path": path, "results": [], "error": str(e)} + # Execute in parallel using joblib results = Parallel( n_jobs=-1, backend="threading", )(delayed(process_single_file)(path) for path in filtered_paths) - + # Collect results and errors matches = [] skipped_file_error_tuples = [] - + for result in results: - if result['error']: - skipped_file_error_tuples.append((result['path'], result['error'])) + if result["error"]: + skipped_file_error_tuples.append((result["path"], result["error"])) else: - matches.extend(result['results']) - + matches.extend(result["results"]) + if skipped_file_error_tuples: log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}") - + log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") - return matches \ No newline at end of file + return matches From c605b1adf557e84b0e135b8a5399776723a01d44 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 17:55:20 +0200 Subject: [PATCH 13/14] Typing --- src/serena/text_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index aa0b2b7..6cab02a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,6 +3,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from typing import Any from joblib import Parallel, delayed from pathspec import PathSpec @@ -262,7 +263,7 @@ def search_files( log.info(f"Processing {len(filtered_paths)} files.") - def process_single_file(path: str): + def process_single_file(path: str) -> dict[str, Any]: """Process a single file - this function will be parallelized.""" try: file_content = file_reader(path) From d13ced7583aa06e66dcd3853462c0cca6a341c0a Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 28 May 2025 18:26:38 +0200 Subject: [PATCH 14/14] Docstring [ci skip] --- src/multilspy/language_server.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 8ec22d0..fab22e5 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -1177,12 +1177,7 @@ class LanguageServer: async def request_parsed_files(self) -> list[str]: - """ - Retrieves relative paths of all files analyzed by the Language Server. - - This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" if not self.server_started: self.logger.log( "request_parsed_files called before Language Server started", @@ -1914,9 +1909,7 @@ class SyncLanguageServer: return self.language_server.retrieve_symbol_body(symbol) def request_parsed_files(self) -> list[str]: - """This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" assert self.loop result = asyncio.run_coroutine_threadsafe( self.language_server.request_parsed_files(), self.loop