mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-05 00:13:01 +00:00
Alex
574f96341e
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
635 lines
22 KiB
Python
635 lines
22 KiB
Python
"""Tests for S3 storage implementation."""
|
|
|
|
import io
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from docsgpt.storage.s3 import S3Storage
|
|
from botocore.exceptions import ClientError
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_boto3_client():
|
|
"""Mock boto3.client to isolate S3 client creation."""
|
|
with patch("boto3.client") as mock_client:
|
|
s3_mock = MagicMock()
|
|
mock_client.return_value = s3_mock
|
|
yield s3_mock
|
|
|
|
|
|
@pytest.fixture
|
|
def s3_storage(mock_boto3_client):
|
|
"""Create S3Storage instance with mocked boto3 client."""
|
|
return S3Storage(bucket_name="test-bucket")
|
|
|
|
|
|
class TestS3StorageInitialization:
|
|
"""Test S3Storage initialization and configuration."""
|
|
|
|
@pytest.mark.unit
|
|
def test_init_with_default_bucket(self):
|
|
"""Should use default bucket name when none provided."""
|
|
with patch("boto3.client"):
|
|
storage = S3Storage()
|
|
assert storage.bucket_name == "docsgpt-test-bucket"
|
|
|
|
@pytest.mark.unit
|
|
def test_init_with_custom_bucket(self):
|
|
"""Should use provided bucket name."""
|
|
with patch("boto3.client"):
|
|
storage = S3Storage(bucket_name="custom-bucket")
|
|
assert storage.bucket_name == "custom-bucket"
|
|
|
|
@pytest.mark.unit
|
|
def test_init_creates_boto3_client_with_s3_credentials(self):
|
|
"""Should create boto3 S3 client with dedicated S3_* credentials."""
|
|
with patch("boto3.client") as mock_client, patch(
|
|
"docsgpt.storage.s3.settings"
|
|
) as mock_settings:
|
|
|
|
mock_settings.S3_BUCKET_NAME = "docsgpt-test-bucket"
|
|
mock_settings.S3_ACCESS_KEY_ID = "test-key"
|
|
mock_settings.S3_SECRET_ACCESS_KEY = "test-secret"
|
|
mock_settings.S3_REGION = "us-west-2"
|
|
mock_settings.S3_ENDPOINT_URL = None
|
|
mock_settings.S3_PATH_STYLE = False
|
|
mock_settings.SAGEMAKER_ACCESS_KEY = None
|
|
mock_settings.SAGEMAKER_SECRET_KEY = None
|
|
mock_settings.SAGEMAKER_REGION = None
|
|
|
|
S3Storage()
|
|
|
|
mock_client.assert_called_once_with(
|
|
"s3",
|
|
aws_access_key_id="test-key",
|
|
aws_secret_access_key="test-secret",
|
|
region_name="us-west-2",
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_init_falls_back_to_sagemaker_credentials(self):
|
|
"""Should fall back to deprecated SAGEMAKER_* credentials when S3_* unset."""
|
|
with patch("boto3.client") as mock_client, patch(
|
|
"docsgpt.storage.s3.settings"
|
|
) as mock_settings, patch("docsgpt.storage.s3.logger") as mock_logger:
|
|
|
|
mock_settings.S3_BUCKET_NAME = "docsgpt-test-bucket"
|
|
mock_settings.S3_ACCESS_KEY_ID = None
|
|
mock_settings.S3_SECRET_ACCESS_KEY = None
|
|
mock_settings.S3_REGION = None
|
|
mock_settings.S3_ENDPOINT_URL = None
|
|
mock_settings.S3_PATH_STYLE = False
|
|
mock_settings.SAGEMAKER_ACCESS_KEY = "legacy-key"
|
|
mock_settings.SAGEMAKER_SECRET_KEY = "legacy-secret"
|
|
mock_settings.SAGEMAKER_REGION = "eu-central-1"
|
|
|
|
S3Storage()
|
|
|
|
mock_client.assert_called_once_with(
|
|
"s3",
|
|
aws_access_key_id="legacy-key",
|
|
aws_secret_access_key="legacy-secret",
|
|
region_name="eu-central-1",
|
|
)
|
|
mock_logger.warning.assert_called_once()
|
|
|
|
@pytest.mark.unit
|
|
def test_init_with_custom_endpoint_and_path_style(self):
|
|
"""Should pass endpoint_url and path-style config for S3-compatible services."""
|
|
with patch("boto3.client") as mock_client, patch(
|
|
"docsgpt.storage.s3.settings"
|
|
) as mock_settings:
|
|
|
|
mock_settings.S3_BUCKET_NAME = "my-bucket"
|
|
mock_settings.S3_ACCESS_KEY_ID = "key"
|
|
mock_settings.S3_SECRET_ACCESS_KEY = "secret"
|
|
mock_settings.S3_REGION = "auto"
|
|
mock_settings.S3_ENDPOINT_URL = "https://account.r2.cloudflarestorage.com"
|
|
mock_settings.S3_PATH_STYLE = True
|
|
mock_settings.SAGEMAKER_ACCESS_KEY = None
|
|
mock_settings.SAGEMAKER_SECRET_KEY = None
|
|
mock_settings.SAGEMAKER_REGION = None
|
|
|
|
S3Storage()
|
|
|
|
_, kwargs = mock_client.call_args
|
|
assert kwargs["endpoint_url"] == "https://account.r2.cloudflarestorage.com"
|
|
assert kwargs["config"].s3 == {"addressing_style": "path"}
|
|
|
|
|
|
class TestS3StorageSaveFile:
|
|
"""Test file saving functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_save_file_uploads_to_s3(self, s3_storage, mock_boto3_client):
|
|
"""Should upload file to S3 with correct parameters."""
|
|
file_data = io.BytesIO(b"test content")
|
|
path = "documents/test.txt"
|
|
|
|
s3_storage.region = "us-east-1"
|
|
result = s3_storage.save_file(file_data, path)
|
|
mock_boto3_client.upload_fileobj.assert_called_once_with(
|
|
file_data,
|
|
"test-bucket",
|
|
path,
|
|
ExtraArgs={"StorageClass": "INTELLIGENT_TIERING"},
|
|
)
|
|
|
|
assert result == {
|
|
"storage_type": "s3",
|
|
"bucket_name": "test-bucket",
|
|
"uri": "s3://test-bucket/documents/test.txt",
|
|
"region": "us-east-1",
|
|
}
|
|
|
|
@pytest.mark.unit
|
|
def test_save_file_with_custom_storage_class(self, s3_storage, mock_boto3_client):
|
|
"""Should use custom storage class when provided."""
|
|
file_data = io.BytesIO(b"test content")
|
|
path = "documents/test.txt"
|
|
|
|
with patch("docsgpt.storage.s3.settings") as mock_settings:
|
|
mock_settings.SAGEMAKER_REGION = "us-east-1"
|
|
s3_storage.save_file(file_data, path, storage_class="STANDARD")
|
|
mock_boto3_client.upload_fileobj.assert_called_once_with(
|
|
file_data, "test-bucket", path, ExtraArgs={"StorageClass": "STANDARD"}
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_save_file_propagates_client_error(self, s3_storage, mock_boto3_client):
|
|
"""Should propagate ClientError when upload fails."""
|
|
file_data = io.BytesIO(b"test content")
|
|
path = "documents/test.txt"
|
|
|
|
mock_boto3_client.upload_fileobj.side_effect = ClientError(
|
|
{"Error": {"Code": "AccessDenied", "Message": "Access denied"}},
|
|
"upload_fileobj",
|
|
)
|
|
|
|
with pytest.raises(ClientError):
|
|
s3_storage.save_file(file_data, path)
|
|
|
|
|
|
class TestS3StorageFileExists:
|
|
"""Test file existence checking."""
|
|
|
|
@pytest.mark.unit
|
|
def test_file_exists_returns_true_when_file_found(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return True when head_object succeeds."""
|
|
path = "documents/test.txt"
|
|
mock_boto3_client.head_object.return_value = {"ContentLength": 100}
|
|
|
|
result = s3_storage.file_exists(path)
|
|
|
|
assert result is True
|
|
mock_boto3_client.head_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key=path
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_file_exists_returns_false_on_client_error(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when head_object raises ClientError."""
|
|
path = "documents/nonexistent.txt"
|
|
mock_boto3_client.head_object.side_effect = ClientError(
|
|
{"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object"
|
|
)
|
|
|
|
result = s3_storage.file_exists(path)
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestS3StorageGetFile:
|
|
"""Test file retrieval functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_get_file_downloads_and_returns_file_object(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should download file from S3 and return BytesIO object."""
|
|
path = "documents/test.txt"
|
|
test_content = b"file content"
|
|
|
|
mock_boto3_client.head_object.return_value = {}
|
|
|
|
def mock_download(bucket, key, file_obj):
|
|
file_obj.write(test_content)
|
|
|
|
mock_boto3_client.download_fileobj.side_effect = mock_download
|
|
|
|
result = s3_storage.get_file(path)
|
|
|
|
assert isinstance(result, io.BytesIO)
|
|
assert result.read() == test_content
|
|
mock_boto3_client.download_fileobj.assert_called_once()
|
|
|
|
@pytest.mark.unit
|
|
def test_get_file_raises_error_when_file_not_found(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should raise FileNotFoundError when file doesn't exist."""
|
|
path = "documents/nonexistent.txt"
|
|
mock_boto3_client.head_object.side_effect = ClientError(
|
|
{"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object"
|
|
)
|
|
|
|
with pytest.raises(FileNotFoundError, match="File not found"):
|
|
s3_storage.get_file(path)
|
|
|
|
|
|
class TestS3StorageFileSize:
|
|
"""Test metadata-only size lookup used by bounded public delivery."""
|
|
|
|
@pytest.mark.unit
|
|
def test_get_file_size_uses_head_without_downloading(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
mock_boto3_client.head_object.return_value = {"ContentLength": 1234}
|
|
|
|
assert s3_storage.get_file_size("avatars/test.png") == 1234
|
|
|
|
mock_boto3_client.head_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key="avatars/test.png"
|
|
)
|
|
mock_boto3_client.download_fileobj.assert_not_called()
|
|
|
|
@pytest.mark.unit
|
|
def test_get_file_size_maps_missing_object_to_file_not_found(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
mock_boto3_client.head_object.side_effect = ClientError(
|
|
{"Error": {"Code": "NoSuchKey", "Message": "Not found"}},
|
|
"head_object",
|
|
)
|
|
|
|
with pytest.raises(FileNotFoundError, match="File not found"):
|
|
s3_storage.get_file_size("avatars/missing.png")
|
|
|
|
@pytest.mark.unit
|
|
def test_get_file_size_maps_denied_head_to_file_not_found(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
# Without s3:ListBucket, HEAD on a missing key returns 403, not 404.
|
|
mock_boto3_client.head_object.side_effect = ClientError(
|
|
{"Error": {"Code": "403", "Message": "Forbidden"}},
|
|
"head_object",
|
|
)
|
|
|
|
with pytest.raises(FileNotFoundError, match="File not found"):
|
|
s3_storage.get_file_size("avatars/missing.png")
|
|
|
|
|
|
class TestS3StoragePresignedUrl:
|
|
"""Test response metadata on short-lived direct avatar delivery URLs."""
|
|
|
|
@pytest.mark.unit
|
|
def test_generate_presigned_url_can_override_content_type(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
mock_boto3_client.generate_presigned_url.return_value = "https://signed"
|
|
|
|
result = s3_storage.generate_presigned_url(
|
|
"avatars/test.png", expires_in=300, content_type="image/png"
|
|
)
|
|
|
|
assert result == "https://signed"
|
|
mock_boto3_client.generate_presigned_url.assert_called_once_with(
|
|
"get_object",
|
|
Params={
|
|
"Bucket": "test-bucket",
|
|
"Key": "avatars/test.png",
|
|
"ResponseContentType": "image/png",
|
|
},
|
|
ExpiresIn=300,
|
|
)
|
|
|
|
|
|
class TestS3StorageDeleteFile:
|
|
"""Test file deletion functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_delete_file_returns_true_on_success(self, s3_storage, mock_boto3_client):
|
|
"""Should return True when deletion succeeds."""
|
|
path = "documents/test.txt"
|
|
mock_boto3_client.delete_object.return_value = {}
|
|
|
|
result = s3_storage.delete_file(path)
|
|
|
|
assert result is True
|
|
mock_boto3_client.delete_object.assert_called_once_with(
|
|
Bucket="test-bucket", Key=path
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_delete_file_returns_false_on_client_error(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when deletion fails with ClientError."""
|
|
path = "documents/test.txt"
|
|
mock_boto3_client.delete_object.side_effect = ClientError(
|
|
{"Error": {"Code": "AccessDenied", "Message": "Access denied"}},
|
|
"delete_object",
|
|
)
|
|
|
|
result = s3_storage.delete_file(path)
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestS3StorageListFiles:
|
|
"""Test directory listing functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_list_files_returns_all_keys_with_prefix(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return all file keys matching the directory prefix."""
|
|
directory = "documents/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [
|
|
{
|
|
"Contents": [
|
|
{"Key": "documents/file1.txt"},
|
|
{"Key": "documents/file2.txt"},
|
|
{"Key": "documents/subdir/file3.txt"},
|
|
]
|
|
}
|
|
]
|
|
|
|
result = s3_storage.list_files(directory)
|
|
|
|
assert len(result) == 3
|
|
assert "documents/file1.txt" in result
|
|
assert "documents/file2.txt" in result
|
|
assert "documents/subdir/file3.txt" in result
|
|
|
|
mock_boto3_client.get_paginator.assert_called_once_with("list_objects_v2")
|
|
paginator_mock.paginate.assert_called_once_with(
|
|
Bucket="test-bucket", Prefix="documents/"
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_list_files_returns_empty_list_when_no_contents(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return empty list when directory has no files."""
|
|
directory = "empty/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [{}]
|
|
|
|
result = s3_storage.list_files(directory)
|
|
|
|
assert result == []
|
|
|
|
|
|
class TestS3StorageProcessFile:
|
|
"""Test file processing functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_process_file_downloads_and_processes_file(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should download file to temp location and call processor function."""
|
|
path = "documents/test.txt"
|
|
|
|
mock_boto3_client.head_object.return_value = {}
|
|
|
|
with patch("tempfile.NamedTemporaryFile") as mock_temp:
|
|
mock_file = MagicMock()
|
|
mock_file.name = "/tmp/test_file"
|
|
mock_temp.return_value.__enter__.return_value = mock_file
|
|
|
|
processor_func = MagicMock(return_value="processed")
|
|
result = s3_storage.process_file(path, processor_func, extra_arg="value")
|
|
assert result == "processed"
|
|
processor_func.assert_called_once_with(
|
|
local_path="/tmp/test_file", extra_arg="value"
|
|
)
|
|
mock_boto3_client.download_fileobj.assert_called_once()
|
|
|
|
@pytest.mark.unit
|
|
def test_process_file_raises_error_when_file_not_found(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should raise FileNotFoundError when file doesn't exist."""
|
|
path = "documents/nonexistent.txt"
|
|
mock_boto3_client.head_object.side_effect = ClientError(
|
|
{"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, "head_object"
|
|
)
|
|
|
|
processor_func = MagicMock()
|
|
|
|
with pytest.raises(FileNotFoundError, match="File not found in S3"):
|
|
s3_storage.process_file(path, processor_func)
|
|
|
|
|
|
class TestS3StorageIsDirectory:
|
|
"""Test directory checking functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_is_directory_returns_true_when_objects_exist(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return True when objects exist with the directory prefix."""
|
|
path = "documents/"
|
|
|
|
mock_boto3_client.list_objects_v2.return_value = {
|
|
"Contents": [{"Key": "documents/file1.txt"}]
|
|
}
|
|
|
|
result = s3_storage.is_directory(path)
|
|
|
|
assert result is True
|
|
mock_boto3_client.list_objects_v2.assert_called_once_with(
|
|
Bucket="test-bucket", Prefix="documents/", MaxKeys=1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_is_directory_returns_false_when_no_objects_exist(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when no objects exist with the directory prefix."""
|
|
path = "nonexistent/"
|
|
|
|
mock_boto3_client.list_objects_v2.return_value = {}
|
|
|
|
result = s3_storage.is_directory(path)
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestS3StorageRemoveDirectory:
|
|
"""Test directory removal functionality."""
|
|
|
|
@pytest.mark.unit
|
|
def test_remove_directory_deletes_all_objects(self, s3_storage, mock_boto3_client):
|
|
"""Should delete all objects with the directory prefix."""
|
|
directory = "documents/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [
|
|
{
|
|
"Contents": [
|
|
{"Key": "documents/file1.txt"},
|
|
{"Key": "documents/file2.txt"},
|
|
]
|
|
}
|
|
]
|
|
|
|
mock_boto3_client.delete_objects.return_value = {
|
|
"Deleted": [{"Key": "documents/file1.txt"}, {"Key": "documents/file2.txt"}]
|
|
}
|
|
|
|
result = s3_storage.remove_directory(directory)
|
|
|
|
assert result is True
|
|
mock_boto3_client.delete_objects.assert_called_once()
|
|
call_args = mock_boto3_client.delete_objects.call_args[1]
|
|
assert call_args["Bucket"] == "test-bucket"
|
|
assert len(call_args["Delete"]["Objects"]) == 2
|
|
|
|
@pytest.mark.unit
|
|
def test_remove_directory_returns_false_when_empty(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when directory is empty (no objects to delete)."""
|
|
directory = "empty/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [{}]
|
|
|
|
result = s3_storage.remove_directory(directory)
|
|
|
|
assert result is False
|
|
mock_boto3_client.delete_objects.assert_not_called()
|
|
|
|
@pytest.mark.unit
|
|
def test_remove_directory_returns_false_on_client_error(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when deletion fails with ClientError."""
|
|
directory = "documents/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [
|
|
{"Contents": [{"Key": "documents/file1.txt"}]}
|
|
]
|
|
|
|
mock_boto3_client.delete_objects.side_effect = ClientError(
|
|
{"Error": {"Code": "AccessDenied", "Message": "Access denied"}},
|
|
"delete_objects",
|
|
)
|
|
|
|
result = s3_storage.remove_directory(directory)
|
|
|
|
assert result is False
|
|
|
|
@pytest.mark.unit
|
|
def test_remove_directory_returns_false_on_delete_errors(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should return False when delete_objects response contains Errors."""
|
|
directory = "documents/"
|
|
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [
|
|
{"Contents": [{"Key": "documents/file1.txt"}]}
|
|
]
|
|
|
|
mock_boto3_client.delete_objects.return_value = {
|
|
"Errors": [{"Key": "documents/file1.txt", "Code": "InternalError"}]
|
|
}
|
|
|
|
result = s3_storage.remove_directory(directory)
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestS3StorageDirectorySlashHandling:
|
|
"""Test that directories without trailing slashes get them added."""
|
|
|
|
@pytest.mark.unit
|
|
def test_list_files_adds_trailing_slash(self, s3_storage, mock_boto3_client):
|
|
"""Should add trailing slash when listing directory without one."""
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [{}]
|
|
|
|
s3_storage.list_files("documents")
|
|
|
|
paginator_mock.paginate.assert_called_once_with(
|
|
Bucket="test-bucket", Prefix="documents/"
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_list_files_empty_directory_string(self, s3_storage, mock_boto3_client):
|
|
"""Empty string directory should not get a slash added."""
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [{}]
|
|
|
|
s3_storage.list_files("")
|
|
|
|
paginator_mock.paginate.assert_called_once_with(
|
|
Bucket="test-bucket", Prefix=""
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_is_directory_adds_trailing_slash(self, s3_storage, mock_boto3_client):
|
|
"""Should add trailing slash for is_directory check."""
|
|
mock_boto3_client.list_objects_v2.return_value = {}
|
|
|
|
s3_storage.is_directory("docs")
|
|
|
|
mock_boto3_client.list_objects_v2.assert_called_once_with(
|
|
Bucket="test-bucket", Prefix="docs/", MaxKeys=1
|
|
)
|
|
|
|
@pytest.mark.unit
|
|
def test_remove_directory_adds_trailing_slash(self, s3_storage, mock_boto3_client):
|
|
"""Should add trailing slash for remove_directory."""
|
|
paginator_mock = MagicMock()
|
|
mock_boto3_client.get_paginator.return_value = paginator_mock
|
|
paginator_mock.paginate.return_value = [{}]
|
|
|
|
s3_storage.remove_directory("docs")
|
|
|
|
paginator_mock.paginate.assert_called_once()
|
|
call_kwargs = paginator_mock.paginate.call_args[1]
|
|
assert call_kwargs["Prefix"] == "docs/"
|
|
|
|
|
|
class TestS3StorageProcessFileError:
|
|
"""Test error handling in process_file."""
|
|
|
|
@pytest.mark.unit
|
|
def test_process_file_propagates_processor_error(
|
|
self, s3_storage, mock_boto3_client
|
|
):
|
|
"""Should propagate errors from the processor function."""
|
|
path = "documents/test.txt"
|
|
mock_boto3_client.head_object.return_value = {}
|
|
|
|
with patch("tempfile.NamedTemporaryFile") as mock_temp:
|
|
mock_file = MagicMock()
|
|
mock_file.name = "/tmp/test_file"
|
|
mock_temp.return_value.__enter__.return_value = mock_file
|
|
|
|
processor_func = MagicMock(side_effect=RuntimeError("Process failed"))
|
|
|
|
with pytest.raises(RuntimeError, match="Process failed"):
|
|
s3_storage.process_file(path, processor_func)
|