diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 0ae262b..ebdafef 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -3,10 +3,10 @@ This script demonstrates how to use Serena's tools locally, useful for testing or development. Here the tools will be operation the serena repo itself. """ -import json from pprint import pprint from serena.agent import * +from serena.constants import REPO_ROOT @dataclass @@ -20,10 +20,10 @@ class InMemorySerenaConfig(SerenaConfigBase): if __name__ == "__main__": - project_path = str(Path("test") / "resources" / "repos" / "python" / "test_repo") - agent = SerenaAgent(project=project_path, serena_config=InMemorySerenaConfig()) + # project_path = str(Path("test") / "resources" / "repos" / "python" / "test_repo") + agent = SerenaAgent(project=REPO_ROOT) # apply a tool - find_symbol_tool = agent.get_tool(FindSymbolTool) - print("Finding the symbol 'VariableContainer'\n") - pprint(json.loads(find_symbol_tool.apply("VariableContainer", within_relative_path=str(Path("test_repo") / "variables.py")))) + find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) + print("Finding the symbol 'SyncLanguageServer'\n") + pprint(json.loads(find_refs_tool.apply(name_path="SyncLanguageServer", relative_file_path="src/multilspy/language_server.py"))) diff --git a/src/serena/agent.py b/src/serena/agent.py index f9771be..da5bf5f 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -37,7 +37,7 @@ from serena.config import SerenaAgentContext, SerenaAgentMode from serena.constants import PROJECT_TEMPLATE_FILE, SERENA_MANAGED_DIR_NAME from serena.dashboard import MemoryLogHandler, SerenaDashboardAPI from serena.prompt_factory import PromptFactory, SerenaPromptFactory -from serena.symbol import SymbolLocation, SymbolManager +from serena.symbol import SymbolManager from serena.text_utils import search_files from serena.util.file_system import scan_directory from serena.util.general import load_yaml, save_yaml @@ -1196,98 +1196,50 @@ class FindReferencingSymbolsTool(Tool): def apply( self, - relative_path: str, - line: int, - column: int, - include_body: bool = False, + name_path: str, + relative_file_path: str, include_kinds: list[int] | None = None, exclude_kinds: list[int] | None = None, max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Finds symbols that reference the symbol at the given location. + Finds symbols that reference the symbol at the given `name_path`. The result will contain metadata about the referencing symbols + as well as a short code snippet around the reference (unless `include_body` is True, then the short snippet will be omitted). Note that among other kinds of references, this function can be used to find (direct) subclasses of a class, as subclasses are referencing symbols that have the kind class. - :param relative_path: the relative path to the file containing the symbol - :param line: the line number - :param column: the column - :param include_body: whether to include the body of the symbols in the result. - Note that this might lead to a very long output, so you should only use this if you actually need the body - of the referencing symbols for the task at hand. Usually it is a better idea to find - the referencing symbols without the body and then use the find_symbol tool to get the body of - specific symbols if needed. - :param include_kinds: an optional list of integers representing the LSP symbol kinds to include. - If provided, only symbols of the given kinds will be included in the result. - Valid kinds: - 1=file, 2=module, 3=namespace, 4=package, 5=class, 6=method, 7=property, 8=field, 9=constructor, 10=enum, - 11=interface, 12=function, 13=variable, 14=constant, 15=string, 16=number, 17=boolean, 18=array, 19=object, - 20=key, 21=null, 22=enum member, 23=struct, 24=event, 25=operator, 26=type parameter - :param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result. - Takes precedence over include_kinds. - :param max_answer_chars: if the output is longer than this number of characters, - no content will be returned. Don't adjust unless there is really no other way to get the content - required for the task. Instead, if the output is too long, you should - make a stricter query. + :param name_path: for finding the symbol to find references for, same logic as in the `find_symbol` tool. + :param relative_file_path: the relative path to the file containing the symbol for which to find references. + :param include_kinds: same as in the `find_symbol` tool. + :param exclude_kinds: same as in the `find_symbol` tool. + :param max_answer_chars: same as in the `find_symbol` tool. :return: a list of JSON objects with the symbols referencing the requested symbol """ + include_body = False # It is probably never a good idea to include the body of the referencing symbols parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None - symbols = self.symbol_manager.find_referencing_symbols_by_location( - SymbolLocation(relative_path, line, column), + references_in_symbols = self.symbol_manager.find_referencing_symbols( + name_path, + relative_file_path=relative_file_path, include_body=include_body, include_kinds=parsed_include_kinds, exclude_kinds=parsed_exclude_kinds, ) - symbol_dicts = [s.to_dict(kind=True, location=True, depth=0, include_body=include_body) for s in symbols] - result = json.dumps(symbol_dicts) + reference_dicts = [] + for ref in references_in_symbols: + ref_dict = ref.symbol.to_dict(kind=True, location=True, depth=0, include_body=include_body) + if not include_body: + ref_relative_path = ref.symbol.location.relative_path + assert ref_relative_path is not None, f"Referencing symbol {ref.symbol.name} has no relative path, this is likely a bug." + content_around_ref = self.language_server.retrieve_content_around_line( + relative_file_path=ref_relative_path, line=ref.line, context_lines_before=1, context_lines_after=1 + ) + ref_dict["content_around_reference"] = content_around_ref.to_display_string() + reference_dicts.append(ref_dict) + result = json.dumps(reference_dicts) return self._limit_length(result, max_answer_chars) -class FindReferencingCodeSnippetsTool(Tool): - """ - Finds code snippets in which the symbol at the given location is referenced. - """ - - def apply( - self, - relative_path: str, - line: int, - column: int, - context_lines_before: int = 0, - context_lines_after: int = 0, - max_answer_chars: int = _DEFAULT_MAX_ANSWER_LENGTH, - ) -> str: - """ - Returns short code snippets where the symbol at the given location is referenced. - - Contrary to the `find_referencing_symbols` tool, this tool returns references that are not symbols but instead - code snippets that may or may not be contained in a symbol (for example, file-level calls). - It may make sense to use this tool to get a quick overview of the code that references - the symbol. Usually, just looking at code snippets is not enough to understand the full context, - unless the case you are investigating is very simple, - or you already have read the relevant symbols using the find_referencing_symbols tool and - now want to get an overview of how the referenced symbol (at the given location) is used in them. - The size of the snippets is controlled by the context_lines_before and context_lines_after parameters. - - :param relative_path: the relative path to the file containing the symbol - :param line: the line number of the symbol to find references for - :param column: the column of the symbol to find references for - :param context_lines_before: the number of lines to include before the line containing the reference - :param context_lines_after: the number of lines to include after the line containing the reference - :param max_answer_chars: if the output is longer than this number of characters, - no content will be returned. Don't adjust unless there is really no other way to get the content - required for the task. Instead, if the output is too long, you should - make a stricter query. - """ - matches = self.language_server.request_references_with_content( - relative_path, line, column, context_lines_before, context_lines_after - ) - result = [match.to_display_string() for match in matches] - result_json_str = json.dumps(result) - return self._limit_length(result_json_str, max_answer_chars) - - class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): """ Replaces the full definition of a symbol. @@ -1295,23 +1247,21 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): def apply( self, + name_path: str, relative_path: str, - line: int, - column: int, body: str, ) -> str: """ - Replaces the body of the symbol at the given location. - Important: Do not try to guess symbol locations but instead use the find_symbol tool to get the correct location. + Replaces the body of the symbol with the given `name_path`. + :param name_path: for finding the symbol to replace, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol - :param line: the line number - :param column: the column :param body: the new symbol body. Important: Provide the correct level of indentation (as the original body). Note that the first line must not be indented (i.e. no leading spaces). """ - self.symbol_manager.replace_body_at_location( - SymbolLocation(relative_path, line, column), + self.symbol_manager.replace_body( + name_path, + relative_file_path=relative_path, body=body, ) return SUCCESS_RESULT @@ -1324,23 +1274,22 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): def apply( self, + name_path: str, relative_path: str, - line: int, - column: int, body: str, ) -> str: """ Inserts the given body/content after the end of the definition of the given symbol (via the symbol's location). A typical use case is to insert a new class, function, method, field or variable assignment. + :param name_path: for finding the symbol to insert after, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol - :param line: the line number - :param column: the column - :param body: the body/content to be inserted + :param body: the body/content to be inserted. Important: the insterted code will automatically have the + same indentation as the symbol's body, so you do not need to provide any indentation. """ - location = SymbolLocation(relative_path, line, column) - self.symbol_manager.insert_after_symbol_at_location( - location, + self.symbol_manager.insert_after_symbol( + name_path, + relative_file_path=relative_path, body=body, ) return SUCCESS_RESULT @@ -1353,9 +1302,8 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): def apply( self, + name_path: str, relative_path: str, - line: int, - column: int, body: str, ) -> str: """ @@ -1363,13 +1311,14 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): A typical use case is to insert a new class, function, method, field or variable assignment. It also can be used to insert a new import statement before the first symbol in the file. + :param name_path: for finding the symbol to insert before, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol - :param line: the line number - :param column: the column - :param body: the body/content to be inserted + :param body: the body/content to be inserted. Important: the insterted code will automatically have the + same indentation as the symbol's body, so you do not need to provide any indentation. """ - self.symbol_manager.insert_before_symbol_at_location( - SymbolLocation(relative_path, line, column), + self.symbol_manager.insert_before_symbol( + name_path, + relative_file_path=relative_path, body=body, ) return SUCCESS_RESULT diff --git a/src/serena/symbol.py b/src/serena/symbol.py index a284a14..9f96df3 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Self, Union from sensai.util.string import ToStringMixin from multilspy import SyncLanguageServer +from multilspy.language_server import ReferenceInSymbol as LSPReferenceInSymbol from multilspy.multilspy_types import Position, SymbolKind, UnifiedSymbolInformation if TYPE_CHECKING: @@ -468,6 +469,24 @@ class Symbol(ToStringMixin): return result +@dataclass +class ReferenceInSymbol(ToStringMixin): + """Same as the class of the same name in the language server, but using Serena's Symbol class. + Be careful to not confuse it with counterpart! + """ + + symbol: Symbol + line: int + character: int + + def get_relative_path(self) -> str | None: + return self.symbol.location.relative_path + + @classmethod + def from_lsp_reference(cls, reference: LSPReferenceInSymbol) -> Self: + return cls(symbol=Symbol(reference.symbol), line=reference.line, character=reference.character) + + class SymbolManager: def __init__(self, lang_server: SyncLanguageServer, agent: Union["SerenaAgent", None] = None) -> None: """ @@ -478,9 +497,6 @@ class SymbolManager: self.lang_server = lang_server self.agent = agent - def _to_symbols(self, items: list[UnifiedSymbolInformation]) -> list[Symbol]: - return [Symbol(s) for s in items] - def find_by_name( self, name_path: str, @@ -527,27 +543,33 @@ class SymbolManager: include_body: bool = False, include_kinds: Sequence[SymbolKind] | None = None, exclude_kinds: Sequence[SymbolKind] | None = None, - ) -> tuple[Symbol, list[Symbol]] | None: + ) -> list[ReferenceInSymbol]: """ Find all symbols that reference the symbol with the given name. If multiple symbols fit the name (e.g. for variables that are overwritten), will use the first one. + + :param name_path: the name path of the symbol to find + :param relative_file_path: the relative path of the file in which the referenced symbol is defined. + :param include_body: whether to include the body of all symbols in the result. + Not recommended, as the referencing symbols will often be files, and thus the bodies will be very long. + :param include_kinds: which kinds of symbols to include in the result. + :param exclude_kinds: which kinds of symbols to exclude from the result. """ symbol_candidates = self.find_by_name(name_path, substring_matching=False, within_relative_path=relative_file_path) if len(symbol_candidates) == 0: log.warning(f"No symbol with name {name_path} found in file {relative_file_path}") - return None + return [] if len(symbol_candidates) > 1: log.error( f"Found {len(symbol_candidates)} symbols with name {name_path} in file {relative_file_path}." f"May be an overwritten variable, in which case you can ignore this error. Proceeding with the first one. " f"Found symbols for {name_path=} in {relative_file_path=}: \n" - f"" + f"{json.dumps([s.location.to_dict() for s in symbol_candidates], indent=2)}" ) symbol = symbol_candidates[0] - referencing_symbols = self.find_referencing_symbols_by_location( + return self.find_referencing_symbols_by_location( symbol.location, include_body=include_body, include_kinds=include_kinds, exclude_kinds=exclude_kinds ) - return symbol, referencing_symbols def find_referencing_symbols_by_location( self, @@ -555,13 +577,14 @@ class SymbolManager: include_body: bool = False, include_kinds: Sequence[SymbolKind] | None = None, exclude_kinds: Sequence[SymbolKind] | None = None, - ) -> list[Symbol]: + ) -> list[ReferenceInSymbol]: """ Find all symbols that reference the symbol at the given location. :param symbol_location: the location of the symbol for which to find references. Does not need to include an end_line, as it is unused in the search. :param include_body: whether to include the body of all symbols in the result. + Not recommended, as the referencing symbols will often be files, and thus the bodies will be very long. Note: you can filter out the bodies of the children if you set include_children_body=False in the to_dict method. :param include_kinds: an optional sequence of ints representing the LSP symbol kind. @@ -575,22 +598,23 @@ class SymbolManager: assert symbol_location.relative_path is not None assert symbol_location.line is not None assert symbol_location.column is not None - symbol_dicts = self.lang_server.request_referencing_symbols( + references = self.lang_server.request_referencing_symbols( relative_file_path=symbol_location.relative_path, line=symbol_location.line, column=symbol_location.column, include_imports=False, include_self=False, include_body=include_body, + include_file_symbols=True, ) if include_kinds is not None: - symbol_dicts = [s for s in symbol_dicts if s["kind"] in include_kinds] + references = [s for s in references if s.symbol["kind"] in include_kinds] if exclude_kinds is not None: - symbol_dicts = [s for s in symbol_dicts if s["kind"] not in exclude_kinds] + references = [s for s in references if s.symbol["kind"] not in exclude_kinds] - return self._to_symbols(symbol_dicts) + return [ReferenceInSymbol.from_lsp_reference(r) for r in references] @contextmanager def _edited_file(self, relative_path: str) -> Iterator[None]: diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 6cab02a..338b295 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,7 +3,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum -from typing import Any +from typing import Any, Self from joblib import Parallel, delayed from pathspec import PathSpec @@ -38,11 +38,16 @@ class TextLine: return " >" return "..." - def format_line(self) -> str: - """Format the line for display with line number and content.""" + def format_line(self, include_line_numbers: bool = True) -> str: + """Format the line for display (e.g.,for logging or passing to an LLM). + + :param include_line_numbers: Whether to include the line number in the result. + """ prefix = self.get_display_prefix() - line_num = str(self.line_number).rjust(4) - return f"{prefix}{line_num}: {self.line_content}" + if include_line_numbers: + line_num = str(self.line_number).rjust(4) + prefix = f"{prefix}{line_num}" + return f"{prefix}:{self.line_content}" @dataclass(kw_only=True) @@ -52,7 +57,7 @@ class MatchedConsecutiveLines: """ lines: list[TextLine] - """All lines in the context of the match. At least one of them should be of match_type MATCH.""" + """All lines in the context of the match. At least one of them is of `match_type` `MATCH`.""" source_file_path: str | None = None """Path to the file where the match was found (Metadata).""" @@ -84,8 +89,27 @@ class MatchedConsecutiveLines: def num_matched_lines(self) -> int: return len(self.matched_lines) - def to_display_string(self) -> str: - return "\n".join([line.format_line() for line in self.lines]) + def to_display_string(self, include_line_numbers: bool = True) -> str: + return "\n".join([line.format_line(include_line_numbers) for line in self.lines]) + + @classmethod + def from_file_contents( + cls, file_contents: str, line: int, context_lines_before: int = 0, context_lines_after: int = 0, source_file_path: str | None = None + ) -> Self: + line_contents = file_contents.split("\n") + start_lineno = max(0, line - context_lines_before) + end_lineno = min(len(line_contents) - 1, line + context_lines_after) + text_lines: list[TextLine] = [] + # before the line + for lineno in range(start_lineno, line): + text_lines.append(TextLine(line_number=lineno, line_content=line_contents[lineno], match_type=LineType.BEFORE_MATCH)) + # the line + text_lines.append(TextLine(line_number=line, line_content=line_contents[line], match_type=LineType.MATCH)) + # after the line + for lineno in range(line + 1, end_lineno + 1): + text_lines.append(TextLine(line_number=lineno, line_content=line_contents[lineno], match_type=LineType.AFTER_MATCH)) + + return cls(lines=text_lines, source_file_path=source_file_path) def search_text( diff --git a/test/multilspy/python/test_symbol_retrieval.py b/test/multilspy/python/test_symbol_retrieval.py index 61a83ce..ba5bd61 100644 --- a/test/multilspy/python/test_symbol_retrieval.py +++ b/test/multilspy/python/test_symbol_retrieval.py @@ -40,7 +40,7 @@ class TestLanguageServerSymbols: """Test request_referencing_symbols for a variable.""" file_path = os.path.join("test_repo", "variables.py") # Line 75 contains the field status that is later modified - ref_symbols = language_server.request_referencing_symbols(file_path, 74, 4) + ref_symbols = [ref.symbol for ref in language_server.request_referencing_symbols(file_path, 74, 4)] assert len(ref_symbols) > 0 ref_lines = [ref["location"]["range"]["start"]["line"] for ref in ref_symbols if "location" in ref and "range" in ref["location"]] @@ -111,7 +111,9 @@ class TestLanguageServerSymbols: if not create_user_symbol or "selectionRange" not in create_user_symbol: raise AssertionError("create_user symbol or its selectionRange not found") sel_start = create_user_symbol["selectionRange"]["start"] - ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ref_symbols = [ + ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ] assert len(ref_symbols) > 0, "No referencing symbols found for create_user (selectionRange)" # Verify the structure of referencing symbols @@ -133,7 +135,9 @@ class TestLanguageServerSymbols: if not user_symbol or "selectionRange" not in user_symbol: raise AssertionError("User symbol or its selectionRange not found") sel_start = user_symbol["selectionRange"]["start"] - ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ref_symbols = [ + ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ] services_references = [ symbol for symbol in ref_symbols @@ -152,7 +156,9 @@ class TestLanguageServerSymbols: if not get_user_symbol or "selectionRange" not in get_user_symbol: raise AssertionError("get_user symbol or its selectionRange not found") sel_start = get_user_symbol["selectionRange"]["start"] - ref_symbols = language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ref_symbols = [ + ref.symbol for ref in language_server.request_referencing_symbols(file_path, sel_start["line"], sel_start["character"]) + ] method_refs = [ symbol for symbol in ref_symbols @@ -169,7 +175,7 @@ class TestLanguageServerSymbols: file_path = os.path.join("test_repo", "services.py") # Line 3 is a blank line or comment try: - ref_symbols = language_server.request_referencing_symbols(file_path, 3, 0) + ref_symbols = [ref.symbol for ref in language_server.request_referencing_symbols(file_path, 3, 0)] # If we get here, make sure we got an empty result assert ref_symbols == [] or ref_symbols is None except Exception: @@ -448,15 +454,19 @@ class TestLanguageServerSymbols: # Get the containing symbol of a variable in a file file_path = os.path.join("test_repo", "services.py") # import of typing - references_to_typing = language_server.request_referencing_symbols( - file_path, 4, 6, include_imports=False, include_file_symbols=True - ) + references_to_typing = [ + ref.symbol + for ref in language_server.request_referencing_symbols(file_path, 4, 6, include_imports=False, include_file_symbols=True) + ] assert {ref["kind"] for ref in references_to_typing} == {SymbolKind.File} assert {ref["body"] for ref in references_to_typing} == {""} # now include bodies - references_to_typing = language_server.request_referencing_symbols( - file_path, 4, 6, include_imports=False, include_file_symbols=True, include_body=True - ) + references_to_typing = [ + ref.symbol + for ref in language_server.request_referencing_symbols( + file_path, 4, 6, include_imports=False, include_file_symbols=True, include_body=True + ) + ] assert {ref["kind"] for ref in references_to_typing} == {SymbolKind.File} assert references_to_typing[0]["body"]