Merge pull request #45 from oraios/feature/tools-extension

CHANGELOG.md

@@ -6,6 +6,8 @@ Changes prior to the next official version change will appear here.
     * bugfix in find_symbol tool (a bug fixed in LS)
     * merged the two overview tools (for dir and file) int a single one
     * one-click setup for Cline enabled
+    * search for pattern tool can now (optionally) search in the entire project
+    * new tool for restarting the language server, in case of other sources of editing apart from Serena
     * Fix `CheckOnboardingPerformedTool`:
         * Tool description was incompatible with project change
         * Returned result was not as useful as it could be (now added list of memories)

README.md

@@ -597,7 +597,8 @@ Here is the full list of Serena's tools with a short description (output of `uv
 * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
 * `replace_lines`: Replaces a range of lines within a file with new content.
 * `replace_symbol_body`: Replaces the full definition of a symbol.
-* `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project.
+* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
+* `search_for_pattern`: Performs a search for a pattern in the project.
 * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
 * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
 * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.

src/multilspy/language_server.py

@@ -26,7 +26,7 @@ from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union,
 
 import pathspec
 
-from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_text
+from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_files, search_text
 from . import multilspy_types
 from .lsp_protocol_handler import lsp_types as LSPTypes
 from .lsp_protocol_handler.lsp_constants import LSPConstants
@@ -265,11 +265,23 @@ class LanguageServer:
                     processed_patterns.append(line.strip())
 
         # Create a pathspec matcher from the processed patterns
-        self.ignore_spec = pathspec.PathSpec.from_lines(
+        self._ignore_spec = pathspec.PathSpec.from_lines(
             pathspec.patterns.GitWildMatchPattern,
             processed_patterns
         )
 
+    def get_ignore_spec(self) -> pathspec.PathSpec:
+        """Returns the pathspec matcher for the paths that were configured to be ignored through
+        the multilspy config file and the .gitignore file.
+
+        This is is a subset of the full language-specific ignore spec that determines
+        which files are relevant for the language server.
+
+        This matcher is useful for operations outside of the language server,
+        such as when searching for relevant non-language files in the project.
+        """
+        return self._ignore_spec
+
     def should_ignore_path(self, relative_path: str) -> bool:
         """
         Determine if a path should be ignored based on file type
@@ -309,7 +321,7 @@ class LanguageServer:
             normalized_path = normalized_path + '/'
 
         # Use the pathspec matcher to check if the path matches any ignore pattern
-        if self.ignore_spec.match_file(normalized_path):
+        if self._ignore_spec.match_file(normalized_path):
             return True
 
         return False
@@ -1166,33 +1178,16 @@ class LanguageServer:
         if isinstance(pattern, str):
             pattern = re.compile(pattern)
 
-        matches = []
-        all_files = await self.request_parsed_files()
-        for path in all_files:
-            # Apply glob filters if provided
-            # TODO: fnmatch is not exactly the same as glob
-            if paths_include_glob and not fnmatch(path, paths_include_glob):
-                self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG)
-                continue
-
-            if paths_exclude_glob and fnmatch(path, paths_exclude_glob):
-                self.logger.log(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}", logging.DEBUG)
-                continue
-
-            file_content = self.retrieve_full_file_content(path)
-            search_results = search_text(
-                pattern,
-                file_content,
-                source_file_path=path,
-                allow_multiline_match=True,
-                context_lines_before=context_lines_before,
-                context_lines_after=context_lines_after
-            )
-            if len(search_results) > 0:
-                self.logger.log(f"Found {len(search_results)} matches in {path}", logging.DEBUG)
-                matches.extend(search_results)
-
-        return matches
+        relative_file_paths = await self.request_parsed_files()
+        return search_files(
+            relative_file_paths,
+            pattern,
+            file_reader=self.retrieve_full_file_content,
+            context_lines_before=context_lines_before,
+            context_lines_after=context_lines_after,
+            paths_include_glob=paths_include_glob,
+            paths_exclude_glob=paths_exclude_glob
+        )
 
     async def request_referencing_symbols(
         self,
@@ -2083,3 +2078,15 @@ class SyncLanguageServer:
         Whether the given path should be ignored.
         """
         return self.language_server.should_ignore_path(relative_path)
+
+    def get_ignore_spec(self) -> pathspec.PathSpec:
+        """Returns the pathspec matcher for the paths that were configured to be ignored through
+        the multilspy config file and the .gitignore file.
+
+        This is is a subset of the full language-specific ignore spec that determines
+        which files are relevant for the language server.
+
+        This matcher is useful for operations outside of the language server,
+        such as when searching for relevant non-language files in the project.
+        """
+        return self.language_server.get_ignore_spec()

src/serena/agent.py

@@ -26,6 +26,7 @@ from serena import serena_root_path, serena_version
 from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler
 from serena.llm.prompt_factory import PromptFactory
 from serena.symbol import SymbolLocation, SymbolManager
+from serena.text_utils import search_files
 from serena.util.class_decorators import singleton
 from serena.util.file_system import scan_directory
 from serena.util.inspection import iter_subclasses
@@ -502,6 +503,20 @@ class ToolMarkerDoesNotRequireActiveProject:
     pass
 
 
+class RestartLanguageServerTool(Tool):
+    """Restarts the language server, may be necessary when edits not through Serena happen."""
+
+    def apply(self) -> str:
+        """Use this tool only on explicit user request or after confirmation.
+        It may be necessary to restart the language server if the user performs edits
+        not through Serena, so the language server state becomes outdated and further editing attempts lead to errors.
+
+        If such editing errors happen, you should suggest using this tool.
+        """
+        self.agent.reset_language_server()
+        return SUCCESS_RESULT
+
+
 class ReadFileTool(Tool):
     """
     Reads a file within the project directory.
@@ -1129,9 +1144,9 @@ class PrepareForNewConversationTool(Tool):
         return self.prompt_factory.create_prepare_for_new_conversation()
 
 
-class SearchInAllCodeTool(Tool):
+class SearchForPatternTool(Tool):
     """
-    Performs a search for a pattern in all code files (and only in code files) in the project.
+    Performs a search for a pattern in the project.
     """
 
     def apply(
@@ -1141,32 +1156,56 @@ class SearchInAllCodeTool(Tool):
         context_lines_after: int = 0,
         paths_include_glob: str | None = None,
         paths_exclude_glob: str | None = None,
+        only_in_code_files: bool = True,
         max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH,
     ) -> str:
         """
-        Search for a pattern in all code files (and only in code files) in the project. Generally, symbolic operations like find_symbol or find_referencing_symbols
+        Search for a pattern in the project. You can select whether all files or only code files should be searched.
+        Generally, symbolic operations like find_symbol or find_referencing_symbols
         should be preferred if you know which symbols you are looking for.
-        If you have to look in non-code files (like notebooks, documentation, etc.), you should use the shell_command tool with grep or similar.
-        This tool can be useful if you are looking for a specific pattern in the codebase that is not a symbol name.
 
         :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string
         :param context_lines_before: Number of lines of context to include before each match
         :param context_lines_after: Number of lines of context to include after each match
         :param paths_include_glob: optional glob pattern specifying files to include in the search; if not provided, search globally.
         :param paths_exclude_glob: optional glob pattern specifying files to exclude from the search (takes precedence over paths_include_glob).
+        :param only_in_code_files: whether to search only in code files or in the entire code base.
+            The explicitly ignored files (from serena config and gitignore) are never searched.
         :param max_answer_chars: if the output is longer than this number of characters,
             no content will be returned. Don't adjust unless there is really no other way to get the content
             required for the task. Instead, if the output is too long, you should
             make a stricter query.
         :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested).
         """
-        matches = self.language_server.search_files_for_pattern(
-            pattern=pattern,
-            context_lines_before=context_lines_before,
-            context_lines_after=context_lines_after,
-            paths_include_glob=paths_include_glob,
-            paths_exclude_glob=paths_exclude_glob,
-        )
+        if only_in_code_files:
+            matches = self.language_server.search_files_for_pattern(
+                pattern=pattern,
+                context_lines_before=context_lines_before,
+                context_lines_after=context_lines_after,
+                paths_include_glob=paths_include_glob,
+                paths_exclude_glob=paths_exclude_glob,
+            )
+        else:
+            # we walk through all files in the project starting from the root
+            files_to_search = []
+            ignore_spec = self.language_server.get_ignore_spec()
+            for root, dirs, files in os.walk(self.project_root):
+                # Don't go into directories that are ignored by modifying dirs inplace
+                # Explanation for the  + "/" part:
+                # pathspec can't handle the matching of directories if they don't end with a slash!
+                # see https://github.com/cpburnz/python-pathspec/issues/89
+                dirs[:] = [d for d in dirs if not ignore_spec.match_file(d + "/")]
+                for file in files:
+                    if not ignore_spec.match_file(os.path.join(root, file)):
+                        files_to_search.append(os.path.join(root, file))
+            # TODO (maybe): not super efficient to walk through the files again and filter if glob patterns are provided
+            #   but it probably never matters and this version required no further refactoring
+            matches = search_files(
+                files_to_search,
+                pattern,
+                paths_include_glob=paths_include_glob,
+                paths_exclude_glob=paths_exclude_glob,
+            )
         # group matches by file
         file_to_matches: dict[str, list[str]] = defaultdict(list)
         for match in matches:

src/serena/text_utils.py

@@ -1,7 +1,14 @@
+import logging
 import re
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import StrEnum
 
+from pathspec import PathSpec
+from pathspec.patterns.gitwildmatch import GitWildMatchPattern
+
+log = logging.getLogger(__name__)
+
 
 class LineType(StrEnum):
     """Enum for different types of lines in search results."""
@@ -205,3 +212,67 @@ def search_text(
                 matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
 
     return matches
+
+
+def default_file_reader(file_path: str) -> str:
+    """Reads using utf-8 encoding."""
+    with open(file_path, encoding="utf-8") as f:
+        return f.read()
+
+
+def search_files(
+    file_paths: list[str],
+    pattern: re.Pattern | str,
+    file_reader: Callable[[str], str] = default_file_reader,
+    context_lines_before: int = 0,
+    context_lines_after: int = 0,
+    paths_include_glob: str | None = None,
+    paths_exclude_glob: str | None = None,
+) -> list[MatchedConsecutiveLines]:
+    """
+    Search for a pattern in a list of files.
+
+    :param file_paths: List of files in which to search
+    :param pattern: Pattern to search for
+    :param file_reader: Function to read a file, by default will just use os.open.
+        All files that can't be read by it will be skipped.
+    :param context_lines_before: Number of context lines to include before matches
+    :param context_lines_after: Number of context lines to include after matches
+    :param paths_include_glob: Optional glob pattern to include files from the list
+    :param paths_exclude_glob: Optional glob pattern to exclude files from the list
+    :return: List of MatchedConsecutiveLines objects
+    """
+    matches = []
+    include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None
+    exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None
+    skipped_file_error_tuples: list[tuple[str, str]] = []
+    for path in file_paths:
+        if include_spec and not include_spec.match_file(path):
+            log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
+            continue
+        if exclude_spec and exclude_spec.match_file(path):
+            log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
+            continue
+        try:
+            file_content = file_reader(path)
+        except Exception as e:
+            skipped_file_error_tuples.append((path, str(e)))
+            continue
+
+        search_results = search_text(
+            pattern,
+            file_content,
+            source_file_path=path,
+            allow_multiline_match=True,
+            context_lines_before=context_lines_before,
+            context_lines_after=context_lines_after,
+        )
+        if len(search_results) > 0:
+            log.debug(f"Found {len(search_results)} matches in {path}")
+            matches.extend(search_results)
+    if skipped_file_error_tuples:
+        log.debug(
+            f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}"
+        )
+
+    return matches

test/serena/test_text_utils.py

@@ -2,10 +2,10 @@ import re
 
 import pytest
 
-from serena.text_utils import LineType, search_text
+from serena.text_utils import LineType, search_files, search_text
 
 
-class TestTextUtils:
+class TestSearchText:
     def test_search_text_with_string_pattern(self):
         """Test searching with a simple string pattern."""
         content = """
@@ -203,3 +203,147 @@ class TestTextUtils:
         # Search with an invalid regex pattern (unmatched parenthesis)
         with pytest.raises(ValueError):
             search_text("example(", content=content)
+
+
+# Mock file reader that always returns matching content
+def mock_reader_always_match(file_path: str) -> str:
+    """Mock file reader that returns content guaranteed to match the simple pattern."""
+    return "This line contains a match."
+
+
+class TestSearchFiles:
+    @pytest.mark.parametrize(
+        "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description",
+        [
+            # Basic cases
+            (["a.py", "b.txt"], "match", None, None, ["a.py", "b.txt"], "No filters"),
+            (["a.py", "b.txt"], "match", "*.py", None, ["a.py"], "Include only .py files"),
+            (["a.py", "b.txt"], "match", None, "*.txt", ["a.py"], "Exclude .txt files"),
+            (["a.py", "b.txt", "c.py"], "match", "*.py", "c.*", ["a.py"], "Include .py, exclude c.*"),
+            # Directory matching - Using pathspec patterns
+            (["main.c", "test/main.c"], "match", "test/*", None, ["test/main.c"], "Include files in test/ subdir"),
+            (["data/a.csv", "data/b.log"], "match", "data/*", "*.log", ["data/a.csv"], "Include data/*, exclude *.log"),
+            (["src/a.py", "tests/b.py"], "match", "src/**", "tests/**", ["src/a.py"], "Include src/**, exclude tests/**"),
+            (["src/mod/a.py", "tests/b.py"], "match", "**/*.py", "tests/**", ["src/mod/a.py"], "Include **/*.py, exclude tests/**"),
+            (["file.py", "dir/file.py"], "match", "dir/*.py", None, ["dir/file.py"], "Include files directly in dir"),
+            (["file.py", "dir/sub/file.py"], "match", "dir/**/*.py", None, ["dir/sub/file.py"], "Include files recursively in dir"),
+            # Overlap and edge cases
+            (["file.py", "dir/file.py"], "match", "*.py", "dir/*", ["file.py"], "Include *.py, exclude files directly in dir"),
+            (["root.py", "adir/a.py", "bdir/b.py"], "match", "a*/*.py", None, ["adir/a.py"], "Include files in dirs starting with 'a'"),
+            (["a.txt", "b.log"], "match", "*.py", None, [], "No files match include pattern"),
+            (["a.py", "b.py"], "match", None, "*.py", [], "All files match exclude pattern"),
+            (["a.py", "b.py"], "match", "a.*", "*.py", [], "Include a.* but exclude *.py -> empty"),
+            (["a.py", "b.py"], "match", "*.py", "b.*", ["a.py"], "Include *.py but exclude b.* -> a.py"),
+        ],
+        ids=lambda x: x if isinstance(x, str) else "",  # Use description as test ID
+    )
+    def test_search_files_include_exclude(
+        self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description
+    ):
+        """
+        Test the include/exclude glob filtering logic in search_files using PathSpec patterns.
+        """
+        results = search_files(
+            file_paths=file_paths,
+            pattern=pattern,
+            file_reader=mock_reader_always_match,
+            paths_include_glob=paths_include_glob,
+            paths_exclude_glob=paths_exclude_glob,
+            context_lines_before=0,  # No context needed for this test focus
+            context_lines_after=0,
+        )
+
+        # Extract the source file paths from the results
+        actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
+
+        # Assert that the matched files are exactly the ones expected
+        assert actual_matched_files == sorted(expected_matched_files)
+
+        # Basic check on results structure if files were expected
+        if expected_matched_files:
+            assert len(results) == len(expected_matched_files)
+            for result in results:
+                assert len(result.matched_lines) == 1  # Mock reader returns one matching line
+                assert result.matched_lines[0].line_content == "This line contains a match."
+                assert result.matched_lines[0].match_type == LineType.MATCH
+
+    def test_search_files_no_pattern_match_in_content(self):
+        """Test that no results are returned if the pattern doesn't match the file content, even if files pass filters."""
+        file_paths = ["a.py", "b.txt"]
+        pattern = "non_existent_pattern_in_mock_content"  # This won't match mock_reader_always_match content
+        results = search_files(
+            file_paths=file_paths,
+            pattern=pattern,
+            file_reader=mock_reader_always_match,  # Content is "This line contains a match."
+            paths_include_glob=None,  # Both files would pass filters
+            paths_exclude_glob=None,
+        )
+        assert len(results) == 0, "Should not find matches if pattern doesn't match content"
+
+    def test_search_files_regex_pattern_with_filters(self):
+        """Test using a regex pattern works correctly along with include/exclude filters."""
+
+        def specific_mock_reader(file_path: str) -> str:
+            # Provide different content for different files to test regex matching
+            if file_path == "a.py":  # noqa: SIM116
+                return "File A: value=123\nFile A: value=456"
+            elif file_path == "b.py":
+                return "File B: value=789"
+            elif file_path == "c.txt":
+                return "File C: value=000"
+            return "No values here."
+
+        file_paths = ["a.py", "b.py", "c.txt"]
+        pattern = re.compile(r"value=(\d+)")  # Regex pattern to find numbers after 'value='
+
+        results = search_files(
+            file_paths=file_paths,
+            pattern=pattern,
+            file_reader=specific_mock_reader,
+            paths_include_glob="*.py",  # Only include .py files
+            paths_exclude_glob="b.*",  # Exclude files starting with b
+        )
+
+        # Expected: a.py included, b.py excluded by glob, c.txt excluded by glob
+        # a.py has two matches for the regex pattern
+        assert len(results) == 2, "Expected 2 matches only from a.py"
+        actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path])
+        assert actual_matched_files == ["a.py", "a.py"], "Both matches should be from a.py"
+        # Check the content of the matched lines
+        assert results[0].matched_lines[0].line_content == "File A: value=123"
+        assert results[1].matched_lines[0].line_content == "File A: value=456"
+
+    def test_search_files_context_lines_with_filters(self):
+        """Test context lines are included correctly when filters are active."""
+
+        def context_mock_reader(file_path: str) -> str:
+            if file_path == "include_me.txt":
+                return "Line before 1\nLine before 2\nMATCH HERE\nLine after 1\nLine after 2"
+            elif file_path == "exclude_me.log":
+                return "Noise\nMATCH HERE\nNoise"
+            return "No match"
+
+        file_paths = ["include_me.txt", "exclude_me.log"]
+        pattern = "MATCH HERE"
+
+        results = search_files(
+            file_paths=file_paths,
+            pattern=pattern,
+            file_reader=context_mock_reader,
+            paths_include_glob="*.txt",  # Only include .txt files
+            paths_exclude_glob=None,
+            context_lines_before=1,
+            context_lines_after=1,
+        )
+
+        # Expected: Only include_me.txt should be processed and matched
+        assert len(results) == 1, "Expected only one result from the included file"
+        result = results[0]
+        assert result.source_file_path == "include_me.txt"
+        assert len(result.lines) == 3, "Expected 3 lines (1 before, 1 match, 1 after)"
+        assert result.lines[0].line_content == "Line before 2", "Incorrect 'before' context line"
+        assert result.lines[0].match_type == LineType.BEFORE_MATCH
+        assert result.lines[1].line_content == "MATCH HERE", "Incorrect 'match' line"
+        assert result.lines[1].match_type == LineType.MATCH
+        assert result.lines[2].line_content == "Line after 1", "Incorrect 'after' context line"
+        assert result.lines[2].match_type == LineType.AFTER_MATCH