From ee90d2f4e85353815dc4919dc9b81e297093f5a1 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 9 Apr 2025 14:43:00 +0200 Subject: [PATCH 1/5] New tool: restart_language_server --- README.md | 1 + src/serena/agent.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 6056b90..f6caedb 100644 --- a/README.md +++ b/README.md @@ -596,6 +596,7 @@ Here is the full list of Serena's tools with a short description (output of `uv * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. * `replace_lines`: Replaces a range of lines within a file with new content. * `replace_symbol_body`: Replaces the full definition of a symbol. +* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. * `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project. * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. diff --git a/src/serena/agent.py b/src/serena/agent.py index e13b1af..19a0ab5 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -502,6 +502,20 @@ class ToolMarkerDoesNotRequireActiveProject: pass +class RestartLanguageServerTool(Tool): + """Restarts the language server, may be necessary when edits not through Serena happen.""" + + def apply(self) -> str: + """Use this tool only on explicit user request or after confirmation. + It may be necessary to restart the language server if the user performs edits + not through Serena, so the language server state becomes outdated and further editing attempts lead to errors. + + If such editing errors happen, you should suggest using this tool. + """ + self.agent.reset_language_server() + return "OK" + + class ReadFileTool(Tool): """ Reads a file within the project directory. From dcc1b7d8f9a773ca2d23ec5ce1089f6526667c02 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 9 Apr 2025 17:32:32 +0200 Subject: [PATCH 2/5] LS, refactoring: separate out search_files so it can be used outside of the LS --- src/multilspy/language_server.py | 39 ++++++++------------------- src/serena/text_utils.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index b298a46..a9d541d 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -26,7 +26,7 @@ from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union, import pathspec -from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_text +from serena.text_utils import LineType, MatchedConsecutiveLines, TextLine, search_files, search_text from . import multilspy_types from .lsp_protocol_handler import lsp_types as LSPTypes from .lsp_protocol_handler.lsp_constants import LSPConstants @@ -1156,34 +1156,17 @@ class LanguageServer: if isinstance(pattern, str): pattern = re.compile(pattern) - matches = [] - all_files = await self.request_parsed_files() - for path in all_files: - # Apply glob filters if provided - # TODO: fnmatch is not exactly the same as glob - if paths_include_glob and not fnmatch(path, paths_include_glob): - self.logger.log(f"Skipping {path}: does not match include pattern {paths_include_glob}", logging.DEBUG) - continue - - if paths_exclude_glob and fnmatch(path, paths_exclude_glob): - self.logger.log(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}", logging.DEBUG) - continue - - file_content = self.retrieve_full_file_content(path) - search_results = search_text( - pattern, - file_content, - source_file_path=path, - allow_multiline_match=True, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after - ) - if len(search_results) > 0: - self.logger.log(f"Found {len(search_results)} matches in {path}", logging.DEBUG) - matches.extend(search_results) + relative_file_paths = await self.request_parsed_files() + return search_files( + relative_file_paths, + pattern, + content_reader=self.retrieve_full_file_content, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob + ) - return matches - async def request_referencing_symbols( self, relative_file_path: str, diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 4c1e146..774b09a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -1,6 +1,11 @@ +import logging import re +from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from fnmatch import fnmatch + +log = logging.getLogger(__name__) class LineType(StrEnum): @@ -205,3 +210,44 @@ def search_text( matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path)) return matches + + +def default_content_reader(file_path: str) -> str: + with open(file_path) as f: + return f.read() + + +def search_files( + file_paths: list[str], + pattern: re.Pattern | str, + content_reader: Callable[[str], str] = default_content_reader, + context_lines_before: int = 0, + context_lines_after: int = 0, + paths_include_glob: str | None = None, + paths_exclude_glob: str | None = None, +) -> list[MatchedConsecutiveLines]: + matches = [] + for path in file_paths: + # TODO: fnmatch is not exactly the same as glob + if paths_include_glob and not fnmatch(path, paths_include_glob): + log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") + continue + + if paths_exclude_glob and fnmatch(path, paths_exclude_glob): + log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") + continue + + file_content = content_reader(path) + search_results = search_text( + pattern, + file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + ) + if len(search_results) > 0: + log.debug(f"Found {len(search_results)} matches in {path}") + matches.extend(search_results) + + return matches From 74584cd9c7c6873417b89fa9020fa176e9e4060f Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 9 Apr 2025 17:52:57 +0200 Subject: [PATCH 3/5] Extended pattern search tool to also work on non-code files --- README.md | 2 +- src/multilspy/language_server.py | 28 ++++++++++++++++-- src/serena/agent.py | 49 ++++++++++++++++++++++++-------- src/serena/text_utils.py | 6 +++- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f6caedb..b8e906f 100644 --- a/README.md +++ b/README.md @@ -597,7 +597,7 @@ Here is the full list of Serena's tools with a short description (output of `uv * `replace_lines`: Replaces a range of lines within a file with new content. * `replace_symbol_body`: Replaces the full definition of a symbol. * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. -* `search_in_all_code`: Performs a search for a pattern in all code files (and only in code files) in the project. +* `search_for_pattern`: Performs a search for a pattern in the project. * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index a9d541d..820c1a5 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -244,10 +244,22 @@ class LanguageServer: processed_patterns.append(line.strip()) # Create a pathspec matcher from the processed patterns - self.ignore_spec = pathspec.PathSpec.from_lines( + self._ignore_spec = pathspec.PathSpec.from_lines( pathspec.patterns.GitWildMatchPattern, processed_patterns ) + + def get_ignore_spec(self) -> pathspec.PathSpec: + """Returns the pathspec matcher for the paths that were configured to be ignored through + the multilspy config file and the .gitignore file. + + This is is a subset of the full language-specific ignore spec that determines + which files are relevant for the language server. + + This matcher is useful for operations outside of the language server, + such as when searching for relevant non-language files in the project. + """ + return self._ignore_spec def should_ignore_path(self, relative_path: str) -> bool: """ @@ -288,7 +300,7 @@ class LanguageServer: normalized_path = normalized_path + '/' # Use the pathspec matcher to check if the path matches any ignore pattern - if self.ignore_spec.match_file(normalized_path): + if self._ignore_spec.match_file(normalized_path): return True return False @@ -2008,3 +2020,15 @@ class SyncLanguageServer: Whether the given path should be ignored. """ return self.language_server.should_ignore_path(relative_path) + + def get_ignore_spec(self) -> pathspec.PathSpec: + """Returns the pathspec matcher for the paths that were configured to be ignored through + the multilspy config file and the .gitignore file. + + This is is a subset of the full language-specific ignore spec that determines + which files are relevant for the language server. + + This matcher is useful for operations outside of the language server, + such as when searching for relevant non-language files in the project. + """ + return self.language_server.get_ignore_spec() diff --git a/src/serena/agent.py b/src/serena/agent.py index 19a0ab5..d8a1f6d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -26,6 +26,7 @@ from serena import serena_root_path, serena_version from serena.gui_log_viewer import GuiLogViewer, GuiLogViewerHandler from serena.llm.prompt_factory import PromptFactory from serena.symbol import SymbolLocation, SymbolManager +from serena.text_utils import search_files from serena.util.class_decorators import singleton from serena.util.file_system import scan_directory from serena.util.inspection import iter_subclasses @@ -1145,9 +1146,9 @@ class PrepareForNewConversationTool(Tool): return self.prompt_factory.create_prepare_for_new_conversation() -class SearchInAllCodeTool(Tool): +class SearchForPatternTool(Tool): """ - Performs a search for a pattern in all code files (and only in code files) in the project. + Performs a search for a pattern in the project. """ def apply( @@ -1157,32 +1158,56 @@ class SearchInAllCodeTool(Tool): context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, + only_in_code_files: bool = True, max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Search for a pattern in all code files (and only in code files) in the project. Generally, symbolic operations like find_symbol or find_referencing_symbols + Search for a pattern in the project. You can select whether all files or only code files should be searched. + Generally, symbolic operations like find_symbol or find_referencing_symbols should be preferred if you know which symbols you are looking for. - If you have to look in non-code files (like notebooks, documentation, etc.), you should use the shell_command tool with grep or similar. - This tool can be useful if you are looking for a specific pattern in the codebase that is not a symbol name. :param pattern: Regular expression pattern to search for, either as a compiled Pattern or string :param context_lines_before: Number of lines of context to include before each match :param context_lines_after: Number of lines of context to include after each match :param paths_include_glob: optional glob pattern specifying files to include in the search; if not provided, search globally. :param paths_exclude_glob: optional glob pattern specifying files to exclude from the search (takes precedence over paths_include_glob). + :param only_in_code_files: whether to search only in code files or in the entire code base. + The explicitly ignored files (from serena config and gitignore) are never searched. :param max_answer_chars: if the output is longer than this number of characters, no content will be returned. Don't adjust unless there is really no other way to get the content required for the task. Instead, if the output is too long, you should make a stricter query. :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested). """ - matches = self.language_server.search_files_for_pattern( - pattern=pattern, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - paths_include_glob=paths_include_glob, - paths_exclude_glob=paths_exclude_glob, - ) + if only_in_code_files: + matches = self.language_server.search_files_for_pattern( + pattern=pattern, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) + else: + # we walk through all files in the project starting from the root + files_to_search = [] + ignore_spec = self.language_server.get_ignore_spec() + for root, dirs, files in os.walk(self.project_root): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not ignore_spec.match_file(d + "/")] + for file in files: + if not ignore_spec.match_file(os.path.join(root, file)): + files_to_search.append(os.path.join(root, file)) + # TODO (maybe): not super efficient to walk through the files again and filter if glob patterns are provided + # but it probably never matters and this version required no further refactoring + matches = search_files( + files_to_search, + pattern, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) # group matches by file file_to_matches: dict[str, list[str]] = defaultdict(list) for match in matches: diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 774b09a..857a826 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -236,8 +236,12 @@ def search_files( if paths_exclude_glob and fnmatch(path, paths_exclude_glob): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + try: + file_content = content_reader(path) + except Exception as e: + log.error(f"Error reading file {path}. Skipping.\nError: {e}") + continue - file_content = content_reader(path) search_results = search_text( pattern, file_content, From 52efc50375cca7653d6b75dd484835141650cc39 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 9 Apr 2025 17:57:47 +0200 Subject: [PATCH 4/5] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19605d7..de3a01f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Changes prior to the next official version change will appear here. * bugfix in find_symbol tool (a bug fixed in LS) * merged the two overview tools (for dir and file) int a single one * one-click setup for Cline enabled + * search for pattern tool can now (optionally) search in the entire project + * new tool for restarting the language server, in case of other sources of editing apart from Serena * Language Servers: * Add further file extensions considered by the language servers for Python (.pyi), JavaScript (.jsx) and TypeScript (.tsx, .jsx) From b6afe6d89dcbb0ba03dfe6946118a9b3a9d3b3d0 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 10 Apr 2025 20:26:44 +0200 Subject: [PATCH 5/5] Use pathspec instead of fnmatch in search_file. Added tests for it Also several small improvements --- src/multilspy/language_server.py | 2 +- src/serena/agent.py | 2 +- src/serena/text_utils.py | 41 ++++++--- test/serena/test_text_utils.py | 148 ++++++++++++++++++++++++++++++- 4 files changed, 179 insertions(+), 14 deletions(-) diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 820c1a5..4d948f3 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -1172,7 +1172,7 @@ class LanguageServer: return search_files( relative_file_paths, pattern, - content_reader=self.retrieve_full_file_content, + file_reader=self.retrieve_full_file_content, context_lines_before=context_lines_before, context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, diff --git a/src/serena/agent.py b/src/serena/agent.py index d8a1f6d..7fce051 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -514,7 +514,7 @@ class RestartLanguageServerTool(Tool): If such editing errors happen, you should suggest using this tool. """ self.agent.reset_language_server() - return "OK" + return SUCCESS_RESULT class ReadFileTool(Tool): diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 857a826..1bae75a 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,7 +3,9 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum -from fnmatch import fnmatch + +from pathspec import PathSpec +from pathspec.patterns.gitwildmatch import GitWildMatchPattern log = logging.getLogger(__name__) @@ -212,34 +214,49 @@ def search_text( return matches -def default_content_reader(file_path: str) -> str: - with open(file_path) as f: +def default_file_reader(file_path: str) -> str: + """Reads using utf-8 encoding.""" + with open(file_path, encoding="utf-8") as f: return f.read() def search_files( file_paths: list[str], pattern: re.Pattern | str, - content_reader: Callable[[str], str] = default_content_reader, + file_reader: Callable[[str], str] = default_file_reader, context_lines_before: int = 0, context_lines_after: int = 0, paths_include_glob: str | None = None, paths_exclude_glob: str | None = None, ) -> list[MatchedConsecutiveLines]: + """ + Search for a pattern in a list of files. + + :param file_paths: List of files in which to search + :param pattern: Pattern to search for + :param file_reader: Function to read a file, by default will just use os.open. + All files that can't be read by it will be skipped. + :param context_lines_before: Number of context lines to include before matches + :param context_lines_after: Number of context lines to include after matches + :param paths_include_glob: Optional glob pattern to include files from the list + :param paths_exclude_glob: Optional glob pattern to exclude files from the list + :return: List of MatchedConsecutiveLines objects + """ matches = [] + include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None + exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None + skipped_file_error_tuples: list[tuple[str, str]] = [] for path in file_paths: - # TODO: fnmatch is not exactly the same as glob - if paths_include_glob and not fnmatch(path, paths_include_glob): + if include_spec and not include_spec.match_file(path): log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") continue - - if paths_exclude_glob and fnmatch(path, paths_exclude_glob): + if exclude_spec and exclude_spec.match_file(path): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue try: - file_content = content_reader(path) + file_content = file_reader(path) except Exception as e: - log.error(f"Error reading file {path}. Skipping.\nError: {e}") + skipped_file_error_tuples.append((path, str(e))) continue search_results = search_text( @@ -253,5 +270,9 @@ def search_files( if len(search_results) > 0: log.debug(f"Found {len(search_results)} matches in {path}") matches.extend(search_results) + if skipped_file_error_tuples: + log.debug( + f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}" + ) return matches diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py index 0092eb9..1fa6929 100644 --- a/test/serena/test_text_utils.py +++ b/test/serena/test_text_utils.py @@ -2,10 +2,10 @@ import re import pytest -from serena.text_utils import LineType, search_text +from serena.text_utils import LineType, search_files, search_text -class TestTextUtils: +class TestSearchText: def test_search_text_with_string_pattern(self): """Test searching with a simple string pattern.""" content = """ @@ -203,3 +203,147 @@ class TestTextUtils: # Search with an invalid regex pattern (unmatched parenthesis) with pytest.raises(ValueError): search_text("example(", content=content) + + +# Mock file reader that always returns matching content +def mock_reader_always_match(file_path: str) -> str: + """Mock file reader that returns content guaranteed to match the simple pattern.""" + return "This line contains a match." + + +class TestSearchFiles: + @pytest.mark.parametrize( + "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description", + [ + # Basic cases + (["a.py", "b.txt"], "match", None, None, ["a.py", "b.txt"], "No filters"), + (["a.py", "b.txt"], "match", "*.py", None, ["a.py"], "Include only .py files"), + (["a.py", "b.txt"], "match", None, "*.txt", ["a.py"], "Exclude .txt files"), + (["a.py", "b.txt", "c.py"], "match", "*.py", "c.*", ["a.py"], "Include .py, exclude c.*"), + # Directory matching - Using pathspec patterns + (["main.c", "test/main.c"], "match", "test/*", None, ["test/main.c"], "Include files in test/ subdir"), + (["data/a.csv", "data/b.log"], "match", "data/*", "*.log", ["data/a.csv"], "Include data/*, exclude *.log"), + (["src/a.py", "tests/b.py"], "match", "src/**", "tests/**", ["src/a.py"], "Include src/**, exclude tests/**"), + (["src/mod/a.py", "tests/b.py"], "match", "**/*.py", "tests/**", ["src/mod/a.py"], "Include **/*.py, exclude tests/**"), + (["file.py", "dir/file.py"], "match", "dir/*.py", None, ["dir/file.py"], "Include files directly in dir"), + (["file.py", "dir/sub/file.py"], "match", "dir/**/*.py", None, ["dir/sub/file.py"], "Include files recursively in dir"), + # Overlap and edge cases + (["file.py", "dir/file.py"], "match", "*.py", "dir/*", ["file.py"], "Include *.py, exclude files directly in dir"), + (["root.py", "adir/a.py", "bdir/b.py"], "match", "a*/*.py", None, ["adir/a.py"], "Include files in dirs starting with 'a'"), + (["a.txt", "b.log"], "match", "*.py", None, [], "No files match include pattern"), + (["a.py", "b.py"], "match", None, "*.py", [], "All files match exclude pattern"), + (["a.py", "b.py"], "match", "a.*", "*.py", [], "Include a.* but exclude *.py -> empty"), + (["a.py", "b.py"], "match", "*.py", "b.*", ["a.py"], "Include *.py but exclude b.* -> a.py"), + ], + ids=lambda x: x if isinstance(x, str) else "", # Use description as test ID + ) + def test_search_files_include_exclude( + self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description + ): + """ + Test the include/exclude glob filtering logic in search_files using PathSpec patterns. + """ + results = search_files( + file_paths=file_paths, + pattern=pattern, + file_reader=mock_reader_always_match, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + context_lines_before=0, # No context needed for this test focus + context_lines_after=0, + ) + + # Extract the source file paths from the results + actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path]) + + # Assert that the matched files are exactly the ones expected + assert actual_matched_files == sorted(expected_matched_files) + + # Basic check on results structure if files were expected + if expected_matched_files: + assert len(results) == len(expected_matched_files) + for result in results: + assert len(result.matched_lines) == 1 # Mock reader returns one matching line + assert result.matched_lines[0].line_content == "This line contains a match." + assert result.matched_lines[0].match_type == LineType.MATCH + + def test_search_files_no_pattern_match_in_content(self): + """Test that no results are returned if the pattern doesn't match the file content, even if files pass filters.""" + file_paths = ["a.py", "b.txt"] + pattern = "non_existent_pattern_in_mock_content" # This won't match mock_reader_always_match content + results = search_files( + file_paths=file_paths, + pattern=pattern, + file_reader=mock_reader_always_match, # Content is "This line contains a match." + paths_include_glob=None, # Both files would pass filters + paths_exclude_glob=None, + ) + assert len(results) == 0, "Should not find matches if pattern doesn't match content" + + def test_search_files_regex_pattern_with_filters(self): + """Test using a regex pattern works correctly along with include/exclude filters.""" + + def specific_mock_reader(file_path: str) -> str: + # Provide different content for different files to test regex matching + if file_path == "a.py": # noqa: SIM116 + return "File A: value=123\nFile A: value=456" + elif file_path == "b.py": + return "File B: value=789" + elif file_path == "c.txt": + return "File C: value=000" + return "No values here." + + file_paths = ["a.py", "b.py", "c.txt"] + pattern = re.compile(r"value=(\d+)") # Regex pattern to find numbers after 'value=' + + results = search_files( + file_paths=file_paths, + pattern=pattern, + file_reader=specific_mock_reader, + paths_include_glob="*.py", # Only include .py files + paths_exclude_glob="b.*", # Exclude files starting with b + ) + + # Expected: a.py included, b.py excluded by glob, c.txt excluded by glob + # a.py has two matches for the regex pattern + assert len(results) == 2, "Expected 2 matches only from a.py" + actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path]) + assert actual_matched_files == ["a.py", "a.py"], "Both matches should be from a.py" + # Check the content of the matched lines + assert results[0].matched_lines[0].line_content == "File A: value=123" + assert results[1].matched_lines[0].line_content == "File A: value=456" + + def test_search_files_context_lines_with_filters(self): + """Test context lines are included correctly when filters are active.""" + + def context_mock_reader(file_path: str) -> str: + if file_path == "include_me.txt": + return "Line before 1\nLine before 2\nMATCH HERE\nLine after 1\nLine after 2" + elif file_path == "exclude_me.log": + return "Noise\nMATCH HERE\nNoise" + return "No match" + + file_paths = ["include_me.txt", "exclude_me.log"] + pattern = "MATCH HERE" + + results = search_files( + file_paths=file_paths, + pattern=pattern, + file_reader=context_mock_reader, + paths_include_glob="*.txt", # Only include .txt files + paths_exclude_glob=None, + context_lines_before=1, + context_lines_after=1, + ) + + # Expected: Only include_me.txt should be processed and matched + assert len(results) == 1, "Expected only one result from the included file" + result = results[0] + assert result.source_file_path == "include_me.txt" + assert len(result.lines) == 3, "Expected 3 lines (1 before, 1 match, 1 after)" + assert result.lines[0].line_content == "Line before 2", "Incorrect 'before' context line" + assert result.lines[0].match_type == LineType.BEFORE_MATCH + assert result.lines[1].line_content == "MATCH HERE", "Incorrect 'match' line" + assert result.lines[1].match_type == LineType.MATCH + assert result.lines[2].line_content == "Line after 1", "Incorrect 'after' context line" + assert result.lines[2].match_type == LineType.AFTER_MATCH