diff --git a/pyproject.toml b/pyproject.toml index 81ca857..02fe562 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "psutil>=7.0.0", "agno>=1.2.15", "docstring_parser>=0.16", + "joblib>=1.5.1", ] [project.scripts] @@ -256,4 +257,4 @@ markers = [ "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", -] \ No newline at end of file +] diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 6ca98b9..8ec22d0 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -136,11 +136,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - from multilspy.language_servers.jedi_language_server.jedi_server import ( - JediServer, - ) + # from multilspy.language_servers.jedi_language_server.jedi_server import ( + # JediServer, + # ) - return JediServer(config, logger, repository_root_path) + # return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -1189,30 +1189,18 @@ class LanguageServer: logging.ERROR, ) raise MultilspyException("Language Server not started") - # TODO: this worked in jedi, but pyright and basedpyright return nothing... - # I don't know why - # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - # symbols = await self.server.send.workspace_symbol(params) or [] - - # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which - # seems to work in all these language servers - # walk through all children recursively, find all symbols of type Module and collect their relative paths - roots = await self.request_full_symbol_tree() - paths = [] - def collect_module_files(symbol): - if symbol["kind"] == multilspy_types.SymbolKind.File: - assert "location" in symbol - paths.append(symbol["location"]["relativePath"]) - - elif symbol["kind"] == multilspy_types.SymbolKind.Package: - for child in symbol["children"]: - collect_module_files(child) - - for root in roots: - collect_module_files(root) - - return paths - + rel_file_paths = [] + for root, dirs, files in os.walk(self.repository_root_path): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + for file in files: + rel_file_path = os.path.join(root, file) + if not self.is_ignored_path(rel_file_path): + rel_file_paths.append(rel_file_path) + return rel_file_paths async def search_files_for_pattern( self, diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 1bae75a..6cab02a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,7 +3,9 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from typing import Any +from joblib import Parallel, delayed from pathspec import PathSpec from pathspec.patterns.gitwildmatch import GitWildMatchPattern @@ -103,7 +105,8 @@ def search_text( content: The text content to search. May be None if source_file_path is provided. source_file_path: Optional path to the source file. If content is None, this has to be passed and the file will be read. - allow_multiline_match: Whether to search across multiple lines + allow_multiline_match: Whether to search across multiple lines. Currently, the default + option (False) is very inefficient, so it is recommended to set this to True. context_lines_before: Number of context lines to include before matches context_lines_after: Number of context lines to include after matches is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") @@ -188,6 +191,8 @@ def search_text( matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path)) else: + # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer, + # it is not urgent to fix, but should be either improved or the option should be removed. # Search line by line for i, line in enumerate(lines): line_num = i + 1 @@ -242,10 +247,11 @@ def search_files( :param paths_exclude_glob: Optional glob pattern to exclude files from the list :return: List of MatchedConsecutiveLines objects """ - matches = [] + # Pre-filter paths (done sequentially to avoid overhead) include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None - skipped_file_error_tuples: list[tuple[str, str]] = [] + + filtered_paths = [] for path in file_paths: if include_spec and not include_spec.match_file(path): log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") @@ -253,26 +259,47 @@ def search_files( if exclude_spec and exclude_spec.match_file(path): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + filtered_paths.append(path) + + log.info(f"Processing {len(filtered_paths)} files.") + + def process_single_file(path: str) -> dict[str, Any]: + """Process a single file - this function will be parallelized.""" try: file_content = file_reader(path) + search_results = search_text( + pattern, + content=file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + ) + if len(search_results) > 0: + log.debug(f"Found {len(search_results)} matches in {path}") + return {"path": path, "results": search_results, "error": None} except Exception as e: - skipped_file_error_tuples.append((path, str(e))) - continue + log.debug(f"Error processing {path}: {e}") + return {"path": path, "results": [], "error": str(e)} + + # Execute in parallel using joblib + results = Parallel( + n_jobs=-1, + backend="threading", + )(delayed(process_single_file)(path) for path in filtered_paths) + + # Collect results and errors + matches = [] + skipped_file_error_tuples = [] + + for result in results: + if result["error"]: + skipped_file_error_tuples.append((result["path"], result["error"])) + else: + matches.extend(result["results"]) - search_results = search_text( - pattern, - file_content, - source_file_path=path, - allow_multiline_match=True, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - ) - if len(search_results) > 0: - log.debug(f"Found {len(search_results)} matches in {path}") - matches.extend(search_results) if skipped_file_error_tuples: - log.debug( - f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}" - ) + log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}") + log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") return matches diff --git a/uv.lock b/uv.lock index eef6518..4314c74 100644 --- a/uv.lock +++ b/uv.lock @@ -455,6 +455,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" }, ] +[[package]] +name = "joblib" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -937,6 +946,7 @@ dependencies = [ { name = "dotenv" }, { name = "fastmcp" }, { name = "jinja2" }, + { name = "joblib" }, { name = "mcp" }, { name = "overrides" }, { name = "pathspec" }, @@ -986,6 +996,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, + { name = "joblib", specifier = ">=1.5.1" }, { name = "mcp", specifier = ">=1.5.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" }, { name = "overrides", specifier = ">=7.7.0,<8" },