mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-03 18:46:54 +00:00
arc53-machine
3511004f50
Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth tokens and client registrations, API keys) into the connection's encrypted envelope, links API-key tools to one connection per distinct credential, allows several accounts per provider, and adds credential_mode to sources and tools. OAuth MCP tools keep resolving each member's own token, as they did before. docsgpt.connectors.service is now the only reader of OAuth tokens: get_valid_token_info refreshes under a row lock and persists rotated refresh tokens, and a revoked grant flags the connection, pauses its sources and notifies the owner. Loaders build from a connection (BaseConnectorLoader.from_connection), so scheduled sync covers Drive, SharePoint and Confluence sources with no browser. S3 and Reddit keys stay on the connection instead of in remote_data. New endpoints: POST /api/connections, /setup, /reconnect, /picker-token, /claim, DELETE /api/connections/<id>, per-action permissions and MCP refresh-tools. Upload, file listing, sync and validate-session take a connection_id; session tokens keep working for this release. The tool executor reads credentials from the resolved connection (owner or member mode) and pauses on a Connect card when a connection needs signing in. docsgpt connectors reencrypt rewrites stored credentials after a key rotation.
205 lines
8.0 KiB
Python
205 lines
8.0 KiB
Python
"""Tests for SharePoint loader."""
|
|
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
|
|
|
|
|
|
def make_response(json_data=None, status_code=200, raise_error=None):
|
|
resp = MagicMock()
|
|
resp.status_code = status_code
|
|
resp.json.return_value = json_data
|
|
resp.content = b"test content"
|
|
if raise_error is not None:
|
|
resp.raise_for_status.side_effect = raise_error
|
|
else:
|
|
resp.raise_for_status.return_value = None
|
|
return resp
|
|
|
|
|
|
class TestSharePointLoaderProcessFile:
|
|
"""Test _process_file method."""
|
|
|
|
def test_size_retrieved_from_root_level(self):
|
|
"""Should retrieve size from root of file_metadata, not nested file object."""
|
|
loader = SharePointLoader.__new__(SharePointLoader)
|
|
|
|
file_metadata = {
|
|
"id": "test-id",
|
|
"name": "test.txt",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-01-01T00:00:00Z",
|
|
"size": 1024,
|
|
"file": {
|
|
"mimeType": "text/plain"
|
|
}
|
|
}
|
|
|
|
doc = loader._process_file(file_metadata, load_content=False)
|
|
|
|
assert doc is not None
|
|
assert doc.extra_info["size"] == 1024
|
|
assert doc.extra_info["file_name"] == "test.txt"
|
|
assert doc.extra_info["mime_type"] == "text/plain"
|
|
|
|
def test_size_null_when_missing(self):
|
|
"""Should return None when size field is missing."""
|
|
loader = SharePointLoader.__new__(SharePointLoader)
|
|
|
|
file_metadata = {
|
|
"id": "test-id",
|
|
"name": "test.txt",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-01-01T00:00:00Z",
|
|
"file": {
|
|
"mimeType": "text/plain"
|
|
}
|
|
}
|
|
|
|
doc = loader._process_file(file_metadata, load_content=False)
|
|
|
|
assert doc is not None
|
|
assert doc.extra_info["size"] is None
|
|
|
|
|
|
class TestSharePointLoaderLoadFileById:
|
|
"""Test _load_file_by_id method."""
|
|
|
|
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
|
@patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1")
|
|
@patch("docsgpt.connectors.service.get_valid_token_info")
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
|
def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
|
"""Should include size field in $select parameter."""
|
|
mock_get_token.return_value = {
|
|
"access_token": "test-token",
|
|
"refresh_token": "test-refresh"
|
|
}
|
|
mock_get.return_value = make_response({
|
|
"id": "test-id",
|
|
"name": "test.txt",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-01-01T00:00:00Z",
|
|
"size": 2048,
|
|
"file": {
|
|
"mimeType": "text/plain"
|
|
}
|
|
})
|
|
|
|
loader = SharePointLoader("test-session")
|
|
doc = loader._load_file_by_id("test-id", load_content=False)
|
|
|
|
assert doc is not None
|
|
assert doc.extra_info["size"] == 2048
|
|
|
|
call_args = mock_get.call_args
|
|
params = call_args[1]["params"]
|
|
assert "size" in params["$select"]
|
|
|
|
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
|
@patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1")
|
|
@patch("docsgpt.connectors.service.get_valid_token_info")
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
|
def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
|
"""Should return document with size from API response."""
|
|
mock_get_token.return_value = {
|
|
"access_token": "test-token",
|
|
"refresh_token": "test-refresh"
|
|
}
|
|
mock_get.return_value = make_response({
|
|
"id": "test-id",
|
|
"name": "document.pdf",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-06-15T10:30:00Z",
|
|
"size": 56789,
|
|
"file": {
|
|
"mimeType": "application/pdf"
|
|
}
|
|
})
|
|
|
|
loader = SharePointLoader("test-session")
|
|
doc = loader._load_file_by_id("test-id", load_content=False)
|
|
|
|
assert doc is not None
|
|
assert doc.doc_id == "test-id"
|
|
assert doc.extra_info["file_name"] == "document.pdf"
|
|
assert doc.extra_info["mime_type"] == "application/pdf"
|
|
assert doc.extra_info["size"] == 56789
|
|
assert doc.extra_info["created_time"] == "2024-01-01T00:00:00Z"
|
|
assert doc.extra_info["modified_time"] == "2024-06-15T10:30:00Z"
|
|
assert doc.extra_info["source"] == "share_point"
|
|
|
|
|
|
class TestSharePointLoaderListItems:
|
|
"""Test _list_items_in_parent method."""
|
|
|
|
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
|
@patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1")
|
|
@patch("docsgpt.connectors.service.get_valid_token_info")
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
|
def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
|
"""Should include size field in $select parameter when listing items."""
|
|
mock_get_token.return_value = {
|
|
"access_token": "test-token",
|
|
"refresh_token": "test-refresh"
|
|
}
|
|
mock_get.return_value = make_response({
|
|
"value": [
|
|
{
|
|
"id": "file-1",
|
|
"name": "file1.txt",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-01-01T00:00:00Z",
|
|
"size": 12345,
|
|
"file": {
|
|
"mimeType": "text/plain"
|
|
}
|
|
}
|
|
]
|
|
})
|
|
|
|
loader = SharePointLoader("test-session")
|
|
docs = loader._list_items_in_parent("parent-id", limit=10, load_content=False)
|
|
|
|
assert len(docs) == 1
|
|
assert docs[0].extra_info["size"] == 12345
|
|
|
|
call_args = mock_get.call_args
|
|
params = call_args[1]["params"]
|
|
assert "size" in params["$select"]
|
|
|
|
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
|
@patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1")
|
|
@patch("docsgpt.connectors.service.get_valid_token_info")
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
|
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
|
def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
|
"""Should include size for folders as well."""
|
|
mock_get_token.return_value = {
|
|
"access_token": "test-token",
|
|
"refresh_token": "test-refresh"
|
|
}
|
|
mock_get.return_value = make_response({
|
|
"value": [
|
|
{
|
|
"id": "folder-1",
|
|
"name": "MyFolder",
|
|
"createdDateTime": "2024-01-01T00:00:00Z",
|
|
"lastModifiedDateTime": "2024-01-01T00:00:00Z",
|
|
"size": 0,
|
|
"folder": {}
|
|
}
|
|
]
|
|
})
|
|
|
|
loader = SharePointLoader("test-session")
|
|
docs = loader._list_items_in_parent("parent-id", limit=10, load_content=False)
|
|
|
|
assert len(docs) == 1
|
|
assert docs[0].extra_info["is_folder"] is True
|
|
assert docs[0].extra_info["size"] == 0
|
|
|