Move "search for pattern" functionality from LS to Project,

adding all necessary related functionality:
  * Handling of ignored files
  * Enumeration of source files

Functions removed from SolidLanguageServer:
  * index_repository
  * request_parsed_files
  * search_files_for_pattern

Implement SearchForPatternTool without language server
This commit is contained in:
Dominik Jain
2025-07-09 22:37:57 +02:00
committed by Dominik Jain
parent 34db0f1446
commit c4d89f21e7
9 changed files with 245 additions and 133 deletions
+27 -7
View File
@@ -17,9 +17,9 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, TypeVar, Union
import click
from pathspec import PathSpec
from sensai.util import logging
from sensai.util.logging import LogTime
from tqdm import tqdm
from serena import serena_version
from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
@@ -140,6 +140,7 @@ def create_ls_for_project(
else:
project_instance = project
# TODO: This is duplicated in Project.__init__
project_config = project_instance.project_config
ignored_paths = project_config.ignored_paths
if len(ignored_paths) > 0:
@@ -178,11 +179,22 @@ def index_project(project: str, log_level: str = "INFO") -> None:
:param project: the project to index. By default, the current working directory is used.
"""
log_level_int = logging.getLevelNamesMapping()[log_level.upper()]
project = os.path.abspath(project)
project_instance = Project.load(os.path.abspath(project))
print(f"Indexing symbols in project {project}")
ls = create_ls_for_project(project, log_level=log_level_int)
save_after_n_files = 10
with ls.start_server():
ls.index_repository()
parsed_files = project_instance.request_parsed_files()
files_processed = 0
pbar = tqdm(parsed_files, disable=False)
for relative_file_path in pbar:
pbar.set_description(f"Indexing ({os.path.basename(relative_file_path)})")
ls.request_document_symbols(relative_file_path, include_body=False)
ls.request_document_symbols(relative_file_path, include_body=True)
files_processed += 1
if files_processed % save_after_n_files == 0:
ls.save_cache()
ls.save_cache()
print(f"Symbols saved to {ls.cache_path}")
@@ -285,8 +297,6 @@ class SerenaAgent:
self.symbol_manager: SymbolManager | None = None
self.memories_manager: MemoriesManager | None = None
self.lines_read: LinesRead | None = None
self.ignore_spec: PathSpec # not set to None to avoid assert statements
"""Ignore spec, extracted from the project's gitignore files and the explicitly configured ignored paths."""
# set the active modes
if modes is None:
@@ -340,6 +350,7 @@ class SerenaAgent:
path = path.resolve()
return path.is_relative_to(_proj_root)
# TODO: Move to project
def path_is_gitignored(self, path: str | Path) -> bool:
"""
Checks if the given path is ignored by git. Non absolute paths are assumed to be relative to the project root.
@@ -354,7 +365,8 @@ class SerenaAgent:
if len(relative_path.parts) > 0 and relative_path.parts[0] == ".git":
return True
return match_path(str(relative_path), self.ignore_spec, root_path=self.get_project_root())
project = self.get_active_project_or_raise()
return match_path(str(relative_path), project.get_ignore_spec(), root_path=self.get_project_root())
def validate_relative_path(self, relative_path: str) -> None:
"""
@@ -384,6 +396,15 @@ class SerenaAgent:
"""
return self._active_project
def get_active_project_or_raise(self) -> Project:
"""
:return: the active project or raises an exception if no project is active
"""
project = self.get_active_project()
if project is None:
raise ValueError("No active project. Please activate a project first.")
return project
def set_modes(self, modes: list[SerenaAgentMode]) -> None:
"""
Set the current mode configurations.
@@ -475,7 +496,6 @@ class SerenaAgent:
with LogTime("Language server initialization", logger=log):
self.reset_language_server()
assert self.language_server is not None
self.ignore_spec = self.language_server.get_ignore_spec()
# initialize project-specific instances which depend on the language server
log.debug(f"Initializing symbol and memories manager for {project.project_name} at {project.project_root}")
+9 -6
View File
@@ -9,7 +9,7 @@ from copy import deepcopy
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from typing import TYPE_CHECKING, Any, Self, TypeVar
from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar
import yaml
from ruamel.yaml.comments import CommentedMap
@@ -23,13 +23,12 @@ from serena.constants import (
SELENA_CONFIG_TEMPLATE_FILE,
SERENA_MANAGED_DIR_NAME,
)
from serena.project import Project
from serena.util.general import load_yaml, save_yaml
from serena.util.inspection import determine_programming_language_composition
from solidlsp.ls_config import Language
if TYPE_CHECKING:
pass
from ..project import Project
log = logging.getLogger(__name__)
T = TypeVar("T")
@@ -233,7 +232,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
For testing purposes, it can also be instantiated directly with the desired parameters.
"""
projects: list[Project] = field(default_factory=list)
projects: list["Project"] = field(default_factory=list)
gui_log_window_enabled: bool = False
log_level: int = logging.INFO
trace_lsp_communication: bool = False
@@ -293,6 +292,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
"""
Static constructor to create SerenaConfig from the configuration file
"""
from ..project import Project
config_file_path = cls._determine_config_file_path()
# create the configuration file from the template if necessary
@@ -387,7 +388,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
def project_names(self) -> list[str]:
return sorted(project.project_config.project_name for project in self.projects)
def get_project(self, project_root_or_name: str) -> Project | None:
def get_project(self, project_root_or_name: str) -> Optional["Project"]:
for project in self.projects:
if project.project_config.project_name == project_root_or_name:
return project
@@ -398,7 +399,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
return project
return None
def add_project_from_path(self, project_root: Path | str, project_name: str | None = None) -> tuple[Project, bool]:
def add_project_from_path(self, project_root: Path | str, project_name: str | None = None) -> tuple["Project", bool]:
"""
Add a project to the Serena configuration from a given path. Will raise a FileExistsError if the
name or path is already registered.
@@ -410,6 +411,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
saved to disk. It may be that no new project configuration was generated if the project configuration already
exists on disk but the project itself was not added yet to the Serena configuration.
"""
from ..project import Project
project_root = Path(project_root).resolve()
if not project_root.exists():
raise FileNotFoundError(f"Error: Path does not exist: {project_root}")
+139 -4
View File
@@ -1,16 +1,45 @@
import logging
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Self
import pathspec
from serena.config.serena_config import ProjectConfig
from serena.text_utils import MatchedConsecutiveLines, search_files
from serena.util.file_system import GitignoreParser, match_path
from solidlsp.ls_config import Language
log = logging.getLogger(__name__)
@dataclass
class Project:
project_root: str
project_config: ProjectConfig
def __init__(self, project_root: str, project_config: ProjectConfig):
self.project_root = project_root
self.project_config = project_config
# gather ignored paths from the project configuration and gitignore files
ignored_paths = project_config.ignored_paths
if len(ignored_paths) > 0:
log.info(f"Using {len(ignored_paths)} ignored paths from the explicit project configuration.")
log.debug(f"Ignored paths: {ignored_paths}")
if project_config.ignore_all_files_in_gitignore:
log.info(f"Parsing all gitignore files in {self.project_root}")
gitignore_parser = GitignoreParser(self.project_root)
log.info(f"Found {len(gitignore_parser.get_ignore_specs())} gitignore files.")
for spec in gitignore_parser.get_ignore_specs():
log.debug(f"Adding {len(spec.patterns)} patterns from {spec.file_path} to the ignored paths.")
ignored_paths.extend(spec.patterns)
# Set up the pathspec matcher for the ignored paths
# for all absolute paths in ignored_paths, convert them to relative paths
processed_patterns = []
for pattern in set(ignored_paths):
# Normalize separators (pathspec expects forward slashes)
pattern = pattern.replace(os.path.sep, "/")
processed_patterns.append(pattern)
log.debug(f"Processing {len(processed_patterns)} ignored paths")
self._ignore_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, processed_patterns)
@property
def project_name(self) -> str:
@@ -42,3 +71,109 @@ class Project:
if not abs_path.exists():
raise FileNotFoundError(f"File not found: {abs_path}")
return abs_path.read_text(encoding=self.project_config.encoding)
def get_ignore_spec(self) -> pathspec.PathSpec:
"""Returns the pathspec matcher for the paths that were configured to be ignored through
the multilspy config.
This is is a subset of the full language-specific ignore spec that determines
which files are relevant for the language server.
This matcher is useful for operations outside of the language server,
such as when searching for relevant non-language files in the project.
"""
return self._ignore_spec
def is_ignored_dirname(self, dirname: str) -> bool:
return dirname.startswith(".")
def is_ignored_path(self, relative_path: str, ignore_unsupported_files: bool = True) -> bool:
"""
Determine whether a path should be ignored based on file type and ignore patterns.
:param relative_path: Relative path to check
:param ignore_unsupported_files: whether files that are not source files shall be ignored
:return: whether the path should be ignored
"""
abs_path = os.path.join(self.project_root, relative_path)
if not os.path.exists(abs_path):
raise FileNotFoundError(f"File {abs_path} not found, the ignore check cannot be performed")
# Check file extension if it's a file
is_file = os.path.isfile(abs_path)
if is_file and ignore_unsupported_files:
fn_matcher = self.language.get_source_fn_matcher()
if not fn_matcher.is_relevant_filename(abs_path):
return True
# Create normalized path for consistent handling
rel_path = Path(relative_path)
# Check each part of the path against always fulfilled ignore conditions
dir_parts = rel_path.parts
if is_file:
dir_parts = dir_parts[:-1]
for part in dir_parts:
if not part: # Skip empty parts (e.g., from leading '/')
continue
if self.is_ignored_dirname(part):
return True
return match_path(relative_path, self.get_ignore_spec(), root_path=self.project_root)
def request_parsed_files(self, relative_path: str = "") -> list[str]:
"""Retrieves relative paths of all files analyzed by the Language Server.
:param relative_path: will only retrieve files that are subpaths of this.
"""
rel_file_paths = []
start_path = os.path.join(self.project_root, relative_path)
if not os.path.exists(start_path):
raise FileNotFoundError(f"Relative path {start_path} not found.")
if os.path.isfile(start_path):
return [relative_path]
else:
for root, dirs, files in os.walk(start_path, followlinks=True):
dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
for file in files:
rel_file_path = os.path.relpath(os.path.join(root, file), start=self.project_root)
try:
if not self.is_ignored_path(rel_file_path):
rel_file_paths.append(rel_file_path)
except FileNotFoundError:
log.warning(
f"File {rel_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files",
)
return rel_file_paths
def search_files_for_pattern(
self,
pattern: str,
relative_path: str = "",
context_lines_before: int = 0,
context_lines_after: int = 0,
paths_include_glob: str | None = None,
paths_exclude_glob: str | None = None,
) -> list[MatchedConsecutiveLines]:
"""
Search for a pattern across all (non-ignored) source files
:param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
:param relative_path:
:param context_lines_before: Number of lines of context to include before each match
:param context_lines_after: Number of lines of context to include after each match
:param paths_include_glob: Glob pattern to filter which files to include in the search
:param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob.
:return: List of matched consecutive lines with context
"""
relative_file_paths = self.request_parsed_files(relative_path=relative_path)
return search_files(
relative_file_paths,
pattern,
root_path=self.project_root,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
paths_include_glob=paths_include_glob,
paths_exclude_glob=paths_exclude_glob,
)
+1 -1
View File
@@ -358,7 +358,7 @@ class SearchForPatternTool(Tool):
raise FileNotFoundError(f"Relative path {relative_path} does not exist.")
if restrict_search_to_code_files:
matches = self.language_server.search_files_for_pattern(
matches = self.project.search_files_for_pattern(
pattern=substring_pattern,
relative_path=relative_path,
context_lines_before=context_lines_before,
+1 -4
View File
@@ -58,10 +58,7 @@ class Component(ABC):
@property
def project(self) -> Project:
project = self.agent.get_active_project()
if project is None:
raise ValueError("No active project")
return project
return self.agent.get_active_project_or_raise()
def create_code_editor(self) -> "CodeEditor":
from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor
+1 -84
View File
@@ -16,9 +16,8 @@ from pathlib import Path, PurePath
from typing import Self, Union, cast
import pathspec
import tqdm
from serena.text_utils import MatchedConsecutiveLines, search_files
from serena.text_utils import MatchedConsecutiveLines
from serena.util.file_system import match_path
from solidlsp import ls_types
from solidlsp.ls_config import Language, LanguageServerConfig
@@ -1190,70 +1189,6 @@ class SolidLanguageServer(ABC):
symbol_body = symbol_body[symbol_start_column:]
return symbol_body
def request_parsed_files(self, relative_path: str = "") -> list[str]:
"""Retrieves relative paths of all files analyzed by the Language Server.
:param relative_path: will only retrieve files that are subpaths of this.
"""
if not self.server_started:
self.logger.log(
"request_parsed_files called before Language Server started",
logging.ERROR,
)
raise LanguageServerException("Language Server not started")
rel_file_paths = []
start_path = os.path.join(self.repository_root_path, relative_path)
if not os.path.exists(start_path):
raise FileNotFoundError(f"Relative path {start_path} not found.")
if os.path.isfile(start_path):
return [relative_path]
else:
for root, dirs, files in os.walk(start_path, followlinks=True):
dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
for file in files:
rel_file_path = os.path.relpath(os.path.join(root, file), start=self.repository_root_path)
try:
if not self.is_ignored_path(rel_file_path):
rel_file_paths.append(rel_file_path)
except FileNotFoundError:
self.logger.log(
f"File {rel_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files",
logging.WARNING,
)
return rel_file_paths
def search_files_for_pattern(
self,
pattern: str,
relative_path: str = "",
context_lines_before: int = 0,
context_lines_after: int = 0,
paths_include_glob: str | None = None,
paths_exclude_glob: str | None = None,
) -> list[MatchedConsecutiveLines]:
"""
Search for a pattern across all files analyzed by the Language Server.
:param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
:param relative_path:
:param context_lines_before: Number of lines of context to include before each match
:param context_lines_after: Number of lines of context to include after each match
:param paths_include_glob: Glob pattern to filter which files to include in the search
:param paths_exclude_glob: Glob pattern to filter which files to exclude from the search. Takes precedence over paths_include_glob.
:return: List of matched consecutive lines with context
"""
relative_file_paths = self.request_parsed_files(relative_path=relative_path)
return search_files(
relative_file_paths,
pattern,
file_reader=self.retrieve_full_file_content,
root_path=self.repository_root_path,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
paths_include_glob=paths_include_glob,
paths_exclude_glob=paths_exclude_glob,
)
def request_referencing_symbols(
self,
relative_file_path: str,
@@ -1584,24 +1519,6 @@ class SolidLanguageServer(ABC):
"""
return Path(self.repository_root_path) / ".serena" / "cache" / self.language_id / "document_symbols_cache_v23-06-25.pkl"
def index_repository(self, progress_bar: bool = True, save_after_n_files: int = 10) -> None:
"""Will go through the entire repository and "index" all files, meaning save their symbols to the cache.
:param progress_bar: Whether to show a progress bar while indexing the repository.
:param save_after_n_files: How many files to process before saving a checkpoint of the cache.
"""
parsed_files = self.request_parsed_files()
files_processed = 0
pbar = tqdm.tqdm(parsed_files, disable=not progress_bar)
for relative_file_path in pbar:
pbar.set_description(f"Indexing ({os.path.basename(relative_file_path)})")
self.request_document_symbols(relative_file_path, include_body=False)
self.request_document_symbols(relative_file_path, include_body=True)
files_processed += 1
if files_processed % save_after_n_files == 0:
self.save_cache()
self.save_cache()
def save_cache(self):
with self._cache_lock:
if not self._cache_has_changed:
+37 -1
View File
@@ -4,6 +4,7 @@ from pathlib import Path
import pytest
from sensai.util.logging import configure
from serena.project import Project
from serena.util.file_system import GitignoreParser
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import Language, LanguageServerConfig
@@ -50,6 +51,11 @@ def create_default_ls(language: Language) -> SolidLanguageServer:
return create_ls(language, repo_path)
def create_default_project(language: Language) -> Project:
repo_path = str(get_repo_path(language))
return Project.load(repo_path)
@pytest.fixture(scope="session")
def repo_path(request: LanguageParamRequest) -> Path:
"""Get the repository path for a specific language.
@@ -73,7 +79,7 @@ def repo_path(request: LanguageParamRequest) -> Path:
@pytest.fixture(scope="session")
def language_server(request: LanguageParamRequest):
"""Create a SyncLanguageServer instance configured for the specified language.
"""Create a language server instance configured for the specified language.
This fixture requires a language parameter via pytest.mark.parametrize:
@@ -104,3 +110,33 @@ def language_server(request: LanguageParamRequest):
yield server
finally:
server.stop()
@pytest.fixture(scope="session")
def project(request: LanguageParamRequest):
"""Create a Project for the specified language.
This fixture requires a language parameter via pytest.mark.parametrize:
Example:
```
@pytest.mark.parametrize("project", [Language.PYTHON], indirect=True)
def test_python_project(project: Project) -> None:
# Use the Python project to test something
pass
```
You can also test multiple languages in a single test:
```
@pytest.mark.parametrize("project", [Language.PYTHON, Language.TYPESCRIPT], indirect=True)
def test_multiple_languages(project: SyncLanguageServer) -> None:
# This test will run once for each language
pass
```
"""
if not hasattr(request, "param"):
raise ValueError("Language parameter must be provided via pytest.mark.parametrize")
language = request.param
yield create_default_project(language)
+19 -16
View File
@@ -1,5 +1,6 @@
import pytest
from serena.project import Project
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import Language
from solidlsp.ls_types import UnifiedSymbolInformation
@@ -120,22 +121,6 @@ class TestLanguageServerBasics:
definition = result[0]
assert definition["relativePath"] == CORE_PATH, "Should find the definition of greet in core.clj"
@pytest.mark.parametrize("language_server", [Language.CLOJURE], indirect=True)
def test_search_files_for_pattern(self, language_server: SolidLanguageServer):
result = language_server.search_files_for_pattern("defn.*greet")
assert result is not None, "Pattern search should return results"
assert len(result) > 0, "Should find at least one match for 'defn.*greet'"
core_matches = [match for match in result if match.source_file_path and "core.clj" in match.source_file_path]
assert len(core_matches) > 0, "Should find greet function in core.clj"
result = language_server.search_files_for_pattern(":require")
assert result is not None, "Should find require statements"
utils_matches = [match for match in result if match.source_file_path and "utils.clj" in match.source_file_path]
assert len(utils_matches) > 0, "Should find require statement in utils.clj"
@pytest.mark.parametrize("language_server", [Language.CLOJURE], indirect=True)
def test_request_references_with_content(self, language_server: SolidLanguageServer):
"""Test references to multiply function with content"""
@@ -214,3 +199,21 @@ class TestLanguageServerBasics:
break
assert found_relevant_references, f"Should have found calculate-area referencing multiply, but got: {result}"
class TestProjectBasics:
@pytest.mark.parametrize("project", [Language.CLOJURE], indirect=True)
def test_search_files_for_pattern(self, project: Project) -> None:
result = project.search_files_for_pattern("defn.*greet")
assert result is not None, "Pattern search should return results"
assert len(result) > 0, "Should find at least one match for 'defn.*greet'"
core_matches = [match for match in result if match.source_file_path and "core.clj" in match.source_file_path]
assert len(core_matches) > 0, "Should find greet function in core.clj"
result = project.search_files_for_pattern(":require")
assert result is not None, "Should find require statements"
utils_matches = [match for match in result if match.source_file_path and "utils.clj" in match.source_file_path]
assert len(utils_matches) > 0, "Should find require statement in utils.clj"
+11 -10
View File
@@ -9,6 +9,7 @@ import os
import pytest
from serena.project import Project
from serena.text_utils import LineType
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language
@@ -185,42 +186,42 @@ class TestLanguageServerBasics:
else:
assert line.match_type == LineType.AFTER_MATCH
@pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True)
def test_search_files_for_pattern(self, language_server: SolidLanguageServer) -> None:
class TestProjectBasics:
@pytest.mark.parametrize("project", [Language.PYTHON], indirect=True)
def test_search_files_for_pattern(self, project: Project) -> None:
"""Test search_files_for_pattern with various patterns and glob filters."""
# Test 1: Search for class definitions across all files
class_pattern = r"class\s+\w+\s*(?:\([^{]*\)|:)"
matches = language_server.search_files_for_pattern(class_pattern)
matches = project.search_files_for_pattern(class_pattern)
assert len(matches) > 0
# Should find multiple classes like User, Item, BaseModel, etc.
assert len(matches) >= 5
# Test 2: Search for specific class with include glob
user_class_pattern = r"class\s+User\s*(?:\([^{]*\)|:)"
matches = language_server.search_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py")
matches = project.search_files_for_pattern(user_class_pattern, paths_include_glob="**/models.py")
assert len(matches) == 1 # Should only find User class in models.py
assert matches[0].source_file_path is not None
assert "models.py" in matches[0].source_file_path
# Test 3: Search for method definitions with exclude glob
method_pattern = r"def\s+\w+\s*\([^)]*\):"
matches = language_server.search_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py")
matches = project.search_files_for_pattern(method_pattern, paths_exclude_glob="**/models.py")
assert len(matches) > 0
# Should find methods in services.py but not in models.py
assert all(match.source_file_path is not None and "models.py" not in match.source_file_path for match in matches)
# Test 4: Search for specific method with both include and exclude globs
create_user_pattern = r"def\s+create_user\s*\([^)]*\)(?:\s*->[^:]+)?:"
matches = language_server.search_files_for_pattern(
create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py"
)
matches = project.search_files_for_pattern(create_user_pattern, paths_include_glob="**/*.py", paths_exclude_glob="**/models.py")
assert len(matches) == 1 # Should only find create_user in services.py
assert matches[0].source_file_path is not None
assert "services.py" in matches[0].source_file_path
# Test 5: Search for a pattern that should appear in multiple files
init_pattern = r"def\s+__init__\s*\([^)]*\):"
matches = language_server.search_files_for_pattern(init_pattern)
matches = project.search_files_for_pattern(init_pattern)
assert len(matches) > 1 # Should find __init__ in multiple classes
# Should find __init__ in both models.py and services.py
assert any(match.source_file_path is not None and "models.py" in match.source_file_path for match in matches)
@@ -228,5 +229,5 @@ class TestLanguageServerBasics:
# Test 6: Search with a pattern that should have no matches
no_match_pattern = r"def\s+this_method_does_not_exist\s*\([^)]*\):"
matches = language_server.search_files_for_pattern(no_match_pattern)
matches = project.search_files_for_pattern(no_match_pattern)
assert len(matches) == 0