tiennm99/serena
1
0
Fork 0
mirror of https://github.com/tiennm99/serena.git synced 2026-09-10 04:20:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #132 from oraios/improve-search-for-pattern-performance

Browse Source 
Improve search for pattern performance
This commit is contained in:
Michael Panchenko
2025-05-28 18:24:10 +02:00
committed by GitHub
parent 0ce2f0d4d6 c605b1adf5
commit 8adb22827b
4 changed files with 75 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+2 -1
View File
@@ -34,6 +34,7 @@ dependencies = [
"psutil>=7.0.0",
"agno>=1.2.15",
"docstring_parser>=0.16",
"joblib>=1.5.1",
]
[project.scripts]
@@ -256,4 +257,4 @@ markers = [
"rust: language server running for Rust",
"typescript: language server running for TypeScript",
"php: language server running for PHP",
]
]
src/multilspy/language_server.py
+16 -28
View File
@@ -136,11 +136,11 @@ class LanguageServer:
return PyrightServer(config, logger, repository_root_path)
# It used to be jedi, but pyright is a bit faster, and also more actively maintained
# Keeping the previous code for reference
from multilspy.language_servers.jedi_language_server.jedi_server import (
JediServer,
)
# from multilspy.language_servers.jedi_language_server.jedi_server import (
# JediServer,
# )
return JediServer(config, logger, repository_root_path)
# return JediServer(config, logger, repository_root_path)
elif config.code_language == Language.JAVA:
from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import (
EclipseJDTLS,
@@ -1189,30 +1189,18 @@ class LanguageServer:
logging.ERROR,
)
raise MultilspyException("Language Server not started")
# TODO: this worked in jedi, but pyright and basedpyright return nothing...
# I don't know why
# params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols
# symbols = await self.server.send.workspace_symbol(params) or []
# Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which
# seems to work in all these language servers
# walk through all children recursively, find all symbols of type Module and collect their relative paths
roots = await self.request_full_symbol_tree()
paths = []
def collect_module_files(symbol):
if symbol["kind"] == multilspy_types.SymbolKind.File:
assert "location" in symbol
paths.append(symbol["location"]["relativePath"])
elif symbol["kind"] == multilspy_types.SymbolKind.Package:
for child in symbol["children"]:
collect_module_files(child)
for root in roots:
collect_module_files(root)
return paths
rel_file_paths = []
for root, dirs, files in os.walk(self.repository_root_path):
# Don't go into directories that are ignored by modifying dirs inplace
# Explanation for the + "/" part:
# pathspec can't handle the matching of directories if they don't end with a slash!
# see https://github.com/cpburnz/python-pathspec/issues/89
dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
for file in files:
rel_file_path = os.path.join(root, file)
if not self.is_ignored_path(rel_file_path):
rel_file_paths.append(rel_file_path)
return rel_file_paths
async def search_files_for_pattern(
self,
src/serena/text_utils.py
+46 -19
View File
@@ -3,7 +3,9 @@ import re
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any
from joblib import Parallel, delayed
from pathspec import PathSpec
from pathspec.patterns.gitwildmatch import GitWildMatchPattern
@@ -103,7 +105,8 @@ def search_text(
content: The text content to search. May be None if source_file_path is provided.
source_file_path: Optional path to the source file. If content is None,
this has to be passed and the file will be read.
allow_multiline_match: Whether to search across multiple lines
allow_multiline_match: Whether to search across multiple lines. Currently, the default
option (False) is very inefficient, so it is recommended to set this to True.
context_lines_before: Number of context lines to include before matches
context_lines_after: Number of context lines to include after matches
is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py")
@@ -188,6 +191,8 @@ def search_text(
matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
else:
# TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer,
# it is not urgent to fix, but should be either improved or the option should be removed.
# Search line by line
for i, line in enumerate(lines):
line_num = i + 1
@@ -242,10 +247,11 @@ def search_files(
:param paths_exclude_glob: Optional glob pattern to exclude files from the list
:return: List of MatchedConsecutiveLines objects
"""
matches = []
# Pre-filter paths (done sequentially to avoid overhead)
include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None
exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None
skipped_file_error_tuples: list[tuple[str, str]] = []
filtered_paths = []
for path in file_paths:
if include_spec and not include_spec.match_file(path):
log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
@@ -253,26 +259,47 @@ def search_files(
if exclude_spec and exclude_spec.match_file(path):
log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
continue
filtered_paths.append(path)
log.info(f"Processing {len(filtered_paths)} files.")
def process_single_file(path: str) -> dict[str, Any]:
"""Process a single file - this function will be parallelized."""
try:
file_content = file_reader(path)
search_results = search_text(
pattern,
content=file_content,
source_file_path=path,
allow_multiline_match=True,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
)
if len(search_results) > 0:
log.debug(f"Found {len(search_results)} matches in {path}")
return {"path": path, "results": search_results, "error": None}
except Exception as e:
skipped_file_error_tuples.append((path, str(e)))
continue
log.debug(f"Error processing {path}: {e}")
return {"path": path, "results": [], "error": str(e)}
# Execute in parallel using joblib
results = Parallel(
n_jobs=-1,
backend="threading",
)(delayed(process_single_file)(path) for path in filtered_paths)
# Collect results and errors
matches = []
skipped_file_error_tuples = []
for result in results:
if result["error"]:
skipped_file_error_tuples.append((result["path"], result["error"]))
else:
matches.extend(result["results"])
search_results = search_text(
pattern,
file_content,
source_file_path=path,
allow_multiline_match=True,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
)
if len(search_results) > 0:
log.debug(f"Found {len(search_results)} matches in {path}")
matches.extend(search_results)
if skipped_file_error_tuples:
log.debug(
f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}"
)
log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}")
log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
return matches
uv.lock
Generated
+11
View File
@@ -455,6 +455,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" },
]
[[package]]
name = "joblib"
version = "1.5.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" },
]
[[package]]
name = "markdown-it-py"
version = "3.0.0"
@@ -937,6 +946,7 @@ dependencies = [
{ name = "dotenv" },
{ name = "fastmcp" },
{ name = "jinja2" },
{ name = "joblib" },
{ name = "mcp" },
{ name = "overrides" },
{ name = "pathspec" },
@@ -986,6 +996,7 @@ requires-dist = [
{ name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" },
{ name = "jinja2", specifier = ">=3.1.6" },
{ name = "jinja2", marker = "extra == 'dev'" },
{ name = "joblib", specifier = ">=1.5.1" },
{ name = "mcp", specifier = ">=1.5.0" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" },
{ name = "overrides", specifier = ">=7.7.0,<8" },