tiennm99/serena
1
0
Fork 0
mirror of https://github.com/tiennm99/serena.git synced 2026-09-03 04:18:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/web-server-logs

Browse Source
This commit is contained in:
Michael Panchenko
2025-05-28 18:37:12 +02:00
parent 9d493d8aab d13ced7583
commit 41df16a0d5
8 changed files with 181 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+61 -1
View File
@@ -1,4 +1,64 @@
#
# macOS specific files
.DS_Store
.AppleDouble
.LSOverride
._*
.Spotlight-V100
.Trashes
Icon
.fseventsd
.DocumentRevisions-V100
.TemporaryItems
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Windows specific files
Thumbs.db
Thumbs.db:encryptable
ehthumbs.db
ehthumbs_vista.db
*.stackdump
[Dd]esktop.ini
$RECYCLE.BIN/
*.cab
*.msi
*.msix
*.msm
*.msp
*.lnk
# Linux specific files
*~
.fuse_hidden*
.directory
.Trash-*
.nfs*
# IDE/Text Editors
# VS Code
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
.history/
# JetBrains IDEs (beyond .idea/)
*.iml
*.ipr
*.iws
out/
.idea_modules/
# Sublime Text
*.tmlanguage.cache
*.tmPreferences.cache
*.stTheme.cache
*.sublime-workspace
*.sublime-project
# Project specific ignore
.idea
temp
pyproject.toml
+2 -1
View File
@@ -36,6 +36,7 @@ dependencies = [
"psutil>=7.0.0",
"agno>=1.2.15",
"docstring_parser>=0.16",
"joblib>=1.5.1",
]
[project.scripts]
@@ -257,4 +258,4 @@ markers = [
"rust: language server running for Rust",
"typescript: language server running for TypeScript",
"php: language server running for PHP",
]
]
src/multilspy/language_server.py
+18 -37
View File
@@ -136,11 +136,11 @@ class LanguageServer:
return PyrightServer(config, logger, repository_root_path)
# It used to be jedi, but pyright is a bit faster, and also more actively maintained
# Keeping the previous code for reference
from multilspy.language_servers.jedi_language_server.jedi_server import (
JediServer,
)
# from multilspy.language_servers.jedi_language_server.jedi_server import (
# JediServer,
# )
return JediServer(config, logger, repository_root_path)
# return JediServer(config, logger, repository_root_path)
elif config.code_language == Language.JAVA:
from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import (
EclipseJDTLS,
@@ -1177,42 +1177,25 @@ class LanguageServer:
async def request_parsed_files(self) -> list[str]:
"""
Retrieves relative paths of all files analyzed by the Language Server.
This is slow, as it finds all files by finding all symbols.
This seems to be the only way, the LSP does not provide any endpoints for listing project files."""
"""Retrieves relative paths of all files analyzed by the Language Server."""
if not self.server_started:
self.logger.log(
"request_parsed_files called before Language Server started",
logging.ERROR,
)
raise MultilspyException("Language Server not started")
# TODO: this worked in jedi, but pyright and basedpyright return nothing...
# I don't know why
# params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols
# symbols = await self.server.send.workspace_symbol(params) or []
# Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which
# seems to work in all these language servers
# walk through all children recursively, find all symbols of type Module and collect their relative paths
roots = await self.request_full_symbol_tree()
paths = []
def collect_module_files(symbol):
if symbol["kind"] == multilspy_types.SymbolKind.File:
assert "location" in symbol
paths.append(symbol["location"]["relativePath"])
elif symbol["kind"] == multilspy_types.SymbolKind.Package:
for child in symbol["children"]:
collect_module_files(child)
for root in roots:
collect_module_files(root)
return paths
rel_file_paths = []
for root, dirs, files in os.walk(self.repository_root_path):
# Don't go into directories that are ignored by modifying dirs inplace
# Explanation for the + "/" part:
# pathspec can't handle the matching of directories if they don't end with a slash!
# see https://github.com/cpburnz/python-pathspec/issues/89
dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
for file in files:
rel_file_path = os.path.join(root, file)
if not self.is_ignored_path(rel_file_path):
rel_file_paths.append(rel_file_path)
return rel_file_paths
async def search_files_for_pattern(
self,
@@ -1926,9 +1909,7 @@ class SyncLanguageServer:
return self.language_server.retrieve_symbol_body(symbol)
def request_parsed_files(self) -> list[str]:
"""This is slow, as it finds all files by finding all symbols.
This seems to be the only way, the LSP does not provide any endpoints for listing project files."""
"""Retrieves relative paths of all files analyzed by the Language Server."""
assert self.loop
result = asyncio.run_coroutine_threadsafe(
self.language_server.request_parsed_files(), self.loop
src/serena/agno.py
+16 -4
View File
@@ -70,20 +70,32 @@ class SerenaAgnoAgentProvider:
load_dotenv()
parser = argparse.ArgumentParser(description="Serena coding assistant")
parser.add_argument(
# Create a mutually exclusive group
group = parser.add_mutually_exclusive_group()
# Add arguments to the group, both pointing to the same destination
group.add_argument(
"--project-file",
required=False,
help="Path to the project (or project.yml file).",
)
group.add_argument(
"--project",
required=False,
help="Path to the project (or project.yml file).",
)
args = parser.parse_args()
if args.project_file:
project_file = Path(args.project_file).resolve()
args_project_file = args.project or args.project_file
if args_project_file:
project_file = Path(args_project_file).resolve()
# If project file path is relative, make it absolute by joining with project root
if not project_file.is_absolute():
# Get the project root directory (parent of scripts directory)
project_root = Path(serena_root_path())
project_file = project_root / args.project_file
project_file = project_root / args_project_file
# Ensure the path is normalized and absolute
project_file = str(project_file.resolve())
src/serena/symbol.py
+25 -7
View File
@@ -20,7 +20,8 @@ log = logging.getLogger(__name__)
@dataclass
class SymbolLocation:
"""
Represents the (start) location of a symbol identifier
Represents the location of a symbol, including the line where the identifier
is defined and the end line of the symbol's body
"""
relative_path: str | None
@@ -37,6 +38,16 @@ class SymbolLocation:
the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
may be None for some types of symbols (e.g. SymbolKind.File)
"""
end_line: int | None = None
"""
the line in which the symbol's body ends. For methods that search based on SymbolLocation,
the end line is not needed, as it is unused by the language server in the search.
However, the end_line will typically be included in the results of search methods,
and thus be provided to the user in the response to such requests.
This is useful for the LLMs, especially the less intelligent ones, as they often fail
to perform symbolic operations and when falling back to line-editing tools, will just make up
an end line.
"""
def __post_init__(self) -> None:
if self.relative_path is not None:
@@ -118,10 +129,7 @@ class Symbol(ToStringMixin):
@property
def location(self) -> SymbolLocation:
"""
:return: the start location of the actual symbol identifier
"""
return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column)
return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line)
@property
def body_start_position(self) -> Position | None:
@@ -147,12 +155,21 @@ class Symbol(ToStringMixin):
@property
def line(self) -> int | None:
"""The line in which the symbol identifier is defined (start line)."""
if "selectionRange" in self.symbol_root:
return self.symbol_root["selectionRange"]["start"]["line"]
else:
# line is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
return None
@property
def end_line(self) -> int | None:
"""The end line of the symbol body, also contained in the `body_end_position`."""
body_end_position = self.body_end_position
if body_end_position is not None:
return body_end_position["line"]
return None
@property
def column(self) -> int | None:
if "selectionRange" in self.symbol_root:
@@ -365,7 +382,8 @@ class SymbolManager:
"""
Find all symbols that reference the symbol at the given location.
:param symbol_location: the location of the symbol for which to find references
:param symbol_location: the location of the symbol for which to find references.
Does not need to include an end_line, as it is unused in the search.
:param include_body: whether to include the body of all symbols in the result.
Note: you can filter out the bodies of the children if you set include_children_body=False
in the to_dict method.
@@ -420,7 +438,7 @@ class SymbolManager:
"""
Replace the body of the symbol at the given location with the given body
:param location: the location of the symbol to replace
:param location: the location of the symbol to replace.
:param body: the new body
"""
# make sure body always ends with at least one newline
src/serena/text_utils.py
+46 -19
View File
@@ -3,7 +3,9 @@ import re
from collections.abc import Callable
from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any
from joblib import Parallel, delayed
from pathspec import PathSpec
from pathspec.patterns.gitwildmatch import GitWildMatchPattern
@@ -103,7 +105,8 @@ def search_text(
content: The text content to search. May be None if source_file_path is provided.
source_file_path: Optional path to the source file. If content is None,
this has to be passed and the file will be read.
allow_multiline_match: Whether to search across multiple lines
allow_multiline_match: Whether to search across multiple lines. Currently, the default
option (False) is very inefficient, so it is recommended to set this to True.
context_lines_before: Number of context lines to include before matches
context_lines_after: Number of context lines to include after matches
is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py")
@@ -188,6 +191,8 @@ def search_text(
matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
else:
# TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer,
# it is not urgent to fix, but should be either improved or the option should be removed.
# Search line by line
for i, line in enumerate(lines):
line_num = i + 1
@@ -242,10 +247,11 @@ def search_files(
:param paths_exclude_glob: Optional glob pattern to exclude files from the list
:return: List of MatchedConsecutiveLines objects
"""
matches = []
# Pre-filter paths (done sequentially to avoid overhead)
include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None
exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None
skipped_file_error_tuples: list[tuple[str, str]] = []
filtered_paths = []
for path in file_paths:
if include_spec and not include_spec.match_file(path):
log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
@@ -253,26 +259,47 @@ def search_files(
if exclude_spec and exclude_spec.match_file(path):
log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
continue
filtered_paths.append(path)
log.info(f"Processing {len(filtered_paths)} files.")
def process_single_file(path: str) -> dict[str, Any]:
"""Process a single file - this function will be parallelized."""
try:
file_content = file_reader(path)
search_results = search_text(
pattern,
content=file_content,
source_file_path=path,
allow_multiline_match=True,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
)
if len(search_results) > 0:
log.debug(f"Found {len(search_results)} matches in {path}")
return {"path": path, "results": search_results, "error": None}
except Exception as e:
skipped_file_error_tuples.append((path, str(e)))
continue
log.debug(f"Error processing {path}: {e}")
return {"path": path, "results": [], "error": str(e)}
# Execute in parallel using joblib
results = Parallel(
n_jobs=-1,
backend="threading",
)(delayed(process_single_file)(path) for path in filtered_paths)
# Collect results and errors
matches = []
skipped_file_error_tuples = []
for result in results:
if result["error"]:
skipped_file_error_tuples.append((result["path"], result["error"]))
else:
matches.extend(result["results"])
search_results = search_text(
pattern,
file_content,
source_file_path=path,
allow_multiline_match=True,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
)
if len(search_results) > 0:
log.debug(f"Found {len(search_results)} matches in {path}")
matches.extend(search_results)
if skipped_file_error_tuples:
log.debug(
f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}"
)
log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}")
log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
return matches
src/serena/util/shell.py
+2 -1
View File
@@ -1,4 +1,5 @@
import os
import platform
import subprocess
from pydantic import BaseModel
@@ -25,7 +26,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr:
process = subprocess.Popen(
command,
shell=True,
shell=platform.system() != "Windows",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE if capture_stderr else None,
text=True,
uv.lock
Generated
+11
View File
@@ -455,6 +455,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" },
]
[[package]]
name = "joblib"
version = "1.5.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" },
]
[[package]]
name = "markdown-it-py"
version = "3.0.0"
@@ -967,6 +976,7 @@ dependencies = [
{ name = "fastapi" },
{ name = "fastmcp" },
{ name = "jinja2" },
{ name = "joblib" },
{ name = "mcp" },
{ name = "overrides" },
{ name = "pathspec" },
@@ -1016,6 +1026,7 @@ requires-dist = [
{ name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" },
{ name = "jinja2", specifier = ">=3.1.6" },
{ name = "jinja2", marker = "extra == 'dev'" },
{ name = "joblib", specifier = ">=1.5.1" },
{ name = "mcp", specifier = ">=1.5.0" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" },
{ name = "overrides", specifier = ">=7.7.0,<8" },