mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-03 18:46:54 +00:00
arc53-machine
c17b23378e
docsgpt/core/settings.py had grown to 258 fields in one 600-line class, touched by about two commits a week, with related settings scattered (GitHub ingest caps inside the embeddings block, API keys in four places, the OpenAI Responses knobs 100 lines from the other OpenAI fields). It is now a package: one module per domain (auth, llm, embeddings, retrieval, vectorstores, database, workers, ingestion, ocr, storage, connectors, server, events, agents, guardrails, scheduler, sandbox, speech), each a SettingsGroup owning its fields and validators, composed by multiple inheritance into the same flat Settings class. Every attribute name, type, default, alias and constraint is unchanged, so settings.NAME reads, .env files and test monkeypatches all keep working; the import path docsgpt.core.settings is the package. Settings.normalize_api_key is kept as a classmethod for callers that reuse it. The comment above or beside each field became its Field(description=...), so the definitions are visible to tooling; the next commit generates the docs reference from them. Pitfall recorded for future groups: pydantic collects validators by method name across the MRO, so two groups naming a validator the same would silently keep only one. Each group's validator has a unique name.
1254 lines
49 KiB
Python
1254 lines
49 KiB
Python
"""
|
|
Tests covering remaining small uncovered-line gaps across many files.
|
|
Each section targets specific uncovered lines identified by coverage analysis.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/wsgi.py (lines 1-5)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestWsgiModule:
|
|
def test_wsgi_imports_app(self):
|
|
"""Verify wsgi.py can be imported and exposes the app object."""
|
|
with patch("docsgpt.app.app") as mock_app:
|
|
mock_app.run = MagicMock()
|
|
import importlib
|
|
import docsgpt.wsgi
|
|
|
|
importlib.reload(docsgpt.wsgi)
|
|
assert hasattr(docsgpt.wsgi, "app")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/celery_init.py (lines 18-20)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestCeleryInitConfigLoggers:
|
|
def test_config_loggers_invokes_setup_logging(self):
|
|
"""Cover lines 18-20: config_loggers signal handler calls setup_logging."""
|
|
with patch(
|
|
"docsgpt.core.logging_config.setup_logging"
|
|
) as mock_setup:
|
|
# The signal handler imports and calls setup_logging from logging_config.
|
|
# We need to ensure the import inside the function resolves to our mock.
|
|
# Re-import and call:
|
|
import importlib
|
|
import docsgpt.celery_init
|
|
|
|
importlib.reload(docsgpt.celery_init)
|
|
# The function body does: from docsgpt.core.logging_config import setup_logging
|
|
# then calls setup_logging(). We need to invoke config_loggers directly.
|
|
# Since it's wrapped by @setup_logging.connect, calling the underlying fn:
|
|
docsgpt.celery_init.config_loggers(None)
|
|
mock_setup.assert_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/docsgpt_provider.py (lines 10, 29, 51)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestDocsGPTProviderLLM:
|
|
def test_init_uses_docsgpt_constants(self):
|
|
"""Cover line 10: DocsGPTAPILLM.__init__ uses DOCSGPT constants."""
|
|
with patch("docsgpt.llm.openai.OpenAILLM.__init__", return_value=None):
|
|
from docsgpt.llm.docsgpt_provider import (
|
|
DocsGPTAPILLM,
|
|
)
|
|
|
|
DocsGPTAPILLM(api_key="test")
|
|
# __init__ called super().__init__ with DOCSGPT_API_KEY
|
|
|
|
def test_raw_gen_delegates_with_docsgpt_model(self):
|
|
"""Cover line 29: _raw_gen calls super with DOCSGPT_MODEL."""
|
|
from docsgpt.llm.docsgpt_provider import DocsGPTAPILLM
|
|
|
|
with patch.object(
|
|
DocsGPTAPILLM.__bases__[0], "_raw_gen", return_value="response"
|
|
) as mock_gen:
|
|
llm = DocsGPTAPILLM.__new__(DocsGPTAPILLM)
|
|
llm._raw_gen(None, "ignored_model", [], stream=False)
|
|
mock_gen.assert_called_once()
|
|
args = mock_gen.call_args
|
|
assert args[0][1] == "docsgpt" # model forced to DOCSGPT_MODEL
|
|
|
|
def test_raw_gen_stream_delegates_with_docsgpt_model(self):
|
|
"""Cover line 51: _raw_gen_stream calls super with DOCSGPT_MODEL."""
|
|
from docsgpt.llm.docsgpt_provider import DocsGPTAPILLM
|
|
|
|
with patch.object(
|
|
DocsGPTAPILLM.__bases__[0], "_raw_gen_stream",
|
|
return_value=iter(["chunk"]),
|
|
) as mock_stream:
|
|
llm = DocsGPTAPILLM.__new__(DocsGPTAPILLM)
|
|
llm._raw_gen_stream(None, "ignored", [], stream=True)
|
|
mock_stream.assert_called_once()
|
|
args = mock_stream.call_args
|
|
assert args[0][1] == "docsgpt"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/agents/tools/base.py (lines 7, 10, 13)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestToolABC:
|
|
def test_cannot_instantiate_tool_abc(self):
|
|
"""Cover lines 7, 10, 13: Tool is abstract."""
|
|
from docsgpt.agents.tools.base import Tool
|
|
|
|
with pytest.raises(TypeError):
|
|
Tool()
|
|
|
|
def test_concrete_subclass_works(self):
|
|
from docsgpt.agents.tools.base import Tool
|
|
|
|
class ConcreteTool(Tool):
|
|
def execute_action(self, action_name, **kwargs):
|
|
return "done"
|
|
|
|
def get_actions_metadata(self):
|
|
return [{"name": "act"}]
|
|
|
|
def get_config_requirements(self):
|
|
return {"key": "val"}
|
|
|
|
t = ConcreteTool()
|
|
assert t.execute_action("act") == "done"
|
|
assert t.get_actions_metadata() == [{"name": "act"}]
|
|
assert t.get_config_requirements() == {"key": "val"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/file/base.py (lines 18-19)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseReaderLoadVectorDocuments:
|
|
def test_load_vector_documents(self):
|
|
"""Cover lines 18-19: BaseReader.load_vector_documents."""
|
|
from docsgpt.parser.file.base import BaseReader
|
|
from docsgpt.parser.schema.base import Document
|
|
|
|
class ConcreteReader(BaseReader):
|
|
def load_data(self, *args, **kwargs):
|
|
return [
|
|
Document(text="hello", extra_info={"k": "v"}),
|
|
Document(text="world"),
|
|
]
|
|
|
|
reader = ConcreteReader()
|
|
lc_docs = reader.load_vector_documents()
|
|
assert len(lc_docs) == 2
|
|
assert lc_docs[0].page_content == "hello"
|
|
assert lc_docs[0].metadata == {"k": "v"}
|
|
assert lc_docs[1].page_content == "world"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/tts/base.py (lines 6, 10)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseTTS:
|
|
def test_cannot_instantiate_base_tts(self):
|
|
"""Cover lines 6, 10: BaseTTS is abstract."""
|
|
from docsgpt.tts.base import BaseTTS
|
|
|
|
with pytest.raises(TypeError):
|
|
BaseTTS()
|
|
|
|
def test_concrete_subclass_works(self):
|
|
from docsgpt.tts.base import BaseTTS
|
|
|
|
class ConcreteTTS(BaseTTS):
|
|
def text_to_speech(self, *args, **kwargs):
|
|
return "audio_data", "en"
|
|
|
|
tts = ConcreteTTS()
|
|
audio, lang = tts.text_to_speech("hello")
|
|
assert audio == "audio_data"
|
|
assert lang == "en"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/retriever/base.py (line 10)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseRetriever:
|
|
def test_cannot_instantiate_base_retriever(self):
|
|
"""Cover line 10: BaseRetriever.search is abstract."""
|
|
from docsgpt.retriever.base import BaseRetriever
|
|
|
|
with pytest.raises(TypeError):
|
|
BaseRetriever()
|
|
|
|
def test_concrete_subclass_works(self):
|
|
from docsgpt.retriever.base import BaseRetriever
|
|
|
|
class ConcreteRetriever(BaseRetriever):
|
|
def search(self, *args, **kwargs):
|
|
return [{"text": "found"}]
|
|
|
|
r = ConcreteRetriever()
|
|
assert r.search("query") == [{"text": "found"}]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/stt/base.py (line 15)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseSTT:
|
|
def test_cannot_instantiate_base_stt(self):
|
|
"""Cover line 15: BaseSTT.transcribe is abstract."""
|
|
from docsgpt.stt.base import BaseSTT
|
|
|
|
with pytest.raises(TypeError):
|
|
BaseSTT()
|
|
|
|
def test_concrete_subclass_works(self):
|
|
from docsgpt.stt.base import BaseSTT
|
|
|
|
class ConcreteSTT(BaseSTT):
|
|
def transcribe(self, file_path, language=None, timestamps=False, diarize=False):
|
|
return {"text": "hello", "language": "en"}
|
|
|
|
s = ConcreteSTT()
|
|
result = s.transcribe(Path("/tmp/test.wav"))
|
|
assert result["text"] == "hello"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/open_router.py (line 9)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestOpenRouterLLM:
|
|
def test_init_uses_openrouter_base_url(self):
|
|
"""Cover line 9: OpenRouterLLM.__init__ delegates to OpenAILLM."""
|
|
from docsgpt.llm.open_router import OpenRouterLLM, OPEN_ROUTER_BASE_URL
|
|
|
|
# Verify the class exists and has the correct base URL constant
|
|
assert OPEN_ROUTER_BASE_URL == "https://openrouter.ai/api/v1"
|
|
assert issubclass(OpenRouterLLM, object)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/groq.py (line 9)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestGroqLLM:
|
|
def test_init_uses_groq_base_url(self):
|
|
"""Cover line 9: GroqLLM.__init__ delegates to OpenAILLM."""
|
|
from docsgpt.llm.groq import GroqLLM, GROQ_BASE_URL
|
|
|
|
# Verify the class exists and has the correct base URL constant
|
|
assert GROQ_BASE_URL == "https://api.groq.com/openai/v1"
|
|
assert issubclass(GroqLLM, object)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/llm_creator.py (line 49)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestLLMCreatorRaisesOnUnknown:
|
|
def test_raises_on_unknown_type(self):
|
|
"""Cover line 49: LLMCreator raises ValueError for unknown type."""
|
|
from docsgpt.llm.llm_creator import LLMCreator
|
|
|
|
with pytest.raises(ValueError, match="No LLM class found"):
|
|
LLMCreator.create_llm(
|
|
"nonexistent_provider_xyz",
|
|
api_key="key",
|
|
user_api_key=None,
|
|
decoded_token={"sub": "test"},
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/storage/storage_creator.py (line 30)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestStorageCreatorRaisesOnUnknown:
|
|
def test_raises_on_unknown_type(self):
|
|
"""Cover line 30: StorageCreator raises ValueError for unknown type."""
|
|
from docsgpt.storage.storage_creator import StorageCreator
|
|
|
|
with pytest.raises(ValueError, match="No storage implementation found"):
|
|
StorageCreator.create_storage("nonexistent_storage_xyz")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/seed/commands.py (line 26)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestSeedCommands:
|
|
def test_seed_main_guard(self):
|
|
"""Cover line 26: __main__ guard in seed/commands.py."""
|
|
# Just verify the module can be imported and has the seed group
|
|
from docsgpt.seed.commands import seed
|
|
|
|
assert seed is not None
|
|
assert hasattr(seed, "name")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/core/json_schema_utils.py (line 26)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestJsonSchemaUtilsGap:
|
|
def test_wrapped_schema_not_dict_raises(self):
|
|
"""Cover line 26: schema field not a dict raises validation error."""
|
|
from docsgpt.core.json_schema_utils import (
|
|
normalize_json_schema_payload,
|
|
JsonSchemaValidationError,
|
|
)
|
|
|
|
with pytest.raises(JsonSchemaValidationError, match="must be a valid JSON object"):
|
|
normalize_json_schema_payload({"schema": "not_a_dict"})
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/stt/upload_limits.py (line 26 - already covered, but ensure path)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestUploadLimitsIsAudioFilename:
|
|
def test_is_audio_filename_returns_false_for_none(self):
|
|
from docsgpt.stt.upload_limits import is_audio_filename
|
|
|
|
assert is_audio_filename(None) is False
|
|
|
|
def test_is_audio_filename_returns_false_for_non_audio(self):
|
|
from docsgpt.stt.upload_limits import is_audio_filename
|
|
|
|
assert is_audio_filename("document.pdf") is False
|
|
|
|
def test_is_audio_filename_returns_true_for_wav(self):
|
|
from docsgpt.stt.upload_limits import is_audio_filename
|
|
|
|
assert is_audio_filename("recording.wav") is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/agents/tools/tool_action_parser.py (line 62)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestToolActionParserGap:
|
|
def test_non_numeric_tool_id_warning(self):
|
|
"""Cover line 62: warning logged when tool_id is not numeric."""
|
|
from docsgpt.agents.tools.tool_action_parser import ToolActionParser
|
|
|
|
parser = ToolActionParser("OpenAILLM")
|
|
# A tool call with a non-numeric tool_id at the end
|
|
call = MagicMock()
|
|
call.name = "some_action_notanumber"
|
|
call.arguments = '{"key": "value"}'
|
|
tool_id, action_name, call_args = parser.parse_args(call)
|
|
assert tool_id == "notanumber"
|
|
assert action_name == "some_action"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/api/answer/services/prompt_renderer.py (line 69)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestPromptRendererGap:
|
|
def test_render_prompt_raises_template_render_error_on_unexpected(self):
|
|
"""Cover line 69: generic exception wrapped in TemplateRenderError."""
|
|
from docsgpt.api.answer.services.prompt_renderer import PromptRenderer
|
|
from docsgpt.templates.template_engine import TemplateRenderError
|
|
|
|
renderer = PromptRenderer()
|
|
|
|
with patch.object(
|
|
renderer, "namespace_manager"
|
|
) as mock_ns:
|
|
mock_ns.build_context.side_effect = RuntimeError("unexpected")
|
|
with pytest.raises(TemplateRenderError, match="Prompt rendering failed"):
|
|
renderer.render_prompt("Hello {{ name }}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/anthropic.py
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestAnthropicLLMRawGen:
|
|
def test_raw_gen_does_not_reenter_the_streaming_orchestrator(self):
|
|
"""``_raw_gen`` used to delegate to ``gen_stream`` when ``stream``
|
|
was truthy, handing it a flattened prompt STRING where a message
|
|
list belongs. It now always makes one Messages API call."""
|
|
import types as _types
|
|
|
|
with patch("docsgpt.llm.anthropic.Anthropic") as MockAnthropic:
|
|
with patch("docsgpt.llm.anthropic.StorageCreator") as MockStorage:
|
|
MockStorage.get_storage.return_value = MagicMock()
|
|
from docsgpt.llm.anthropic import AnthropicLLM
|
|
|
|
client = MockAnthropic.return_value
|
|
client.messages.create.return_value = _types.SimpleNamespace(
|
|
content=[_types.SimpleNamespace(type="text", text="answer")],
|
|
stop_reason="end_turn",
|
|
usage=None,
|
|
)
|
|
|
|
llm = AnthropicLLM(api_key="test_key")
|
|
llm.gen_stream = MagicMock()
|
|
messages = [
|
|
{"role": "system", "content": "context"},
|
|
{"role": "user", "content": "question"},
|
|
]
|
|
result = llm._raw_gen(None, "claude-x", messages, stream=True)
|
|
|
|
llm.gen_stream.assert_not_called()
|
|
client.messages.create.assert_called_once()
|
|
assert result == "answer"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/llm/base.py (line 201)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseLLMAbstractRawGen:
|
|
def test_raw_gen_is_abstract(self):
|
|
"""Cover line 201: _raw_gen abstract pass."""
|
|
from docsgpt.llm.base import BaseLLM
|
|
|
|
with pytest.raises(TypeError):
|
|
BaseLLM()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/core/settings (normalize_api_key)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestSettingsNormalizeApiKey:
|
|
def test_normalize_api_key_none_str_returns_none(self):
|
|
"""Cover line 184+: normalize_api_key converts 'None' string to None."""
|
|
from docsgpt.core.settings import Settings
|
|
|
|
result = Settings.normalize_api_key("None")
|
|
assert result is None
|
|
|
|
def test_normalize_api_key_empty_returns_none(self):
|
|
from docsgpt.core.settings import Settings
|
|
|
|
result = Settings.normalize_api_key("")
|
|
assert result is None
|
|
|
|
def test_normalize_api_key_returns_stripped_value(self):
|
|
from docsgpt.core.settings import Settings
|
|
|
|
result = Settings.normalize_api_key(" hello ")
|
|
assert result == "hello"
|
|
|
|
def test_normalize_api_key_non_str_returns_as_is(self):
|
|
from docsgpt.core.settings import Settings
|
|
|
|
result = Settings.normalize_api_key(42)
|
|
assert result == 42
|
|
|
|
def test_normalize_api_key_none_returns_none(self):
|
|
from docsgpt.core.settings import Settings
|
|
|
|
result = Settings.normalize_api_key(None)
|
|
assert result is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/agents/workflow_agent.py (line 43)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestWorkflowAgentGen:
|
|
def test_gen_yields_from_inner(self):
|
|
"""Cover line 43: gen method yields from _gen_inner."""
|
|
with patch(
|
|
"docsgpt.agents.workflow_agent.WorkflowAgent.__init__",
|
|
return_value=None,
|
|
):
|
|
from docsgpt.agents.workflow_agent import WorkflowAgent
|
|
|
|
agent = WorkflowAgent.__new__(WorkflowAgent)
|
|
agent._gen_inner = MagicMock(
|
|
return_value=iter([{"type": "text", "content": "hi"}])
|
|
)
|
|
result = list(agent.gen("hello"))
|
|
assert result == [{"type": "text", "content": "hi"}]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/templates/namespaces.py (line 16)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestNamespaceBuilderABC:
|
|
def test_cannot_instantiate_namespace_builder(self):
|
|
"""Cover line 16: NamespaceBuilder is abstract."""
|
|
from docsgpt.templates.namespaces import NamespaceBuilder
|
|
|
|
with pytest.raises(TypeError):
|
|
NamespaceBuilder()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/file/markdown_parser.py (line 67)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestMarkdownParserEmptyHeader:
|
|
def test_empty_text_header_continues(self):
|
|
"""Cover line 67: when current_text is empty string, continue."""
|
|
from docsgpt.parser.file.markdown_parser import MarkdownParser
|
|
|
|
parser = MarkdownParser()
|
|
# Two consecutive headers with no text between them
|
|
content = "# Header 1\n# Header 2\nSome content"
|
|
# Call the internal method directly
|
|
tups = parser.markdown_to_tups(content)
|
|
# The first header has empty text, so it should be skipped (continue)
|
|
# Only Header 2 with "Some content" should remain
|
|
assert len(tups) >= 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/core/url_validation.py (lines 89-90)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestUrlValidationResolveHostname:
|
|
def test_resolve_hostname_failure_returns_none(self):
|
|
"""Cover lines 89-90: socket.gaierror returns None."""
|
|
import socket
|
|
from docsgpt.core.url_validation import resolve_hostname
|
|
|
|
with patch("socket.gethostbyname", side_effect=socket.gaierror):
|
|
result = resolve_hostname("nonexistent.invalid")
|
|
assert result is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/api/user/agents/webhooks.py (line 72)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestWebhookEmptyPayloadWarning:
|
|
def test_enqueue_with_empty_payload_logs_warning(self):
|
|
"""Cover line 72: empty payload triggers warning log."""
|
|
from flask import Flask
|
|
|
|
app = Flask(__name__)
|
|
with app.test_request_context():
|
|
from docsgpt.api.user.agents.webhooks import AgentWebhookListener
|
|
|
|
resource = AgentWebhookListener()
|
|
with patch.object(
|
|
app.logger, "warning"
|
|
) as mock_warn:
|
|
with patch.object(app.logger, "info"):
|
|
with patch(
|
|
"docsgpt.api.user.agents.webhooks.process_agent_webhook"
|
|
) as mock_task:
|
|
mock_task.delay.return_value = MagicMock(id="task123")
|
|
resource._enqueue_webhook_task("agent123", {}, "POST")
|
|
mock_warn.assert_called_once()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/agents/tools/duckduckgo.py (lines 25-27)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestDuckDuckGoGetClient:
|
|
def test_get_ddgs_client(self):
|
|
"""Cover lines 25-27: _get_ddgs_client imports and returns DDGS."""
|
|
with patch.dict("sys.modules", {"ddgs": MagicMock()}):
|
|
from docsgpt.agents.tools.duckduckgo import DuckDuckGoSearchTool
|
|
|
|
tool = DuckDuckGoSearchTool({"timeout": 10})
|
|
client = tool._get_ddgs_client()
|
|
assert client is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/agents/tools/read_webpage.py (lines 54-55)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestReadWebpageErrors:
|
|
def test_generic_error_returns_error_message(self):
|
|
"""Cover lines 54-55: generic Exception returns error string."""
|
|
from docsgpt.agents.tools.read_webpage import ReadWebpageTool
|
|
|
|
tool = ReadWebpageTool({})
|
|
with patch(
|
|
"docsgpt.agents.tools.read_webpage.pinned_fetch_bytes"
|
|
) as mock_fetch:
|
|
mock_response = MagicMock()
|
|
mock_response.raise_for_status.return_value = None
|
|
mock_response.headers = {"Content-Type": "text/html"}
|
|
mock_fetch.return_value = (b"<html><body>test</body></html>", mock_response)
|
|
with patch(
|
|
"docsgpt.agents.tools.read_webpage.markdownify",
|
|
side_effect=Exception("parse error"),
|
|
):
|
|
result = tool.execute_action("read", url="https://example.com")
|
|
assert "Error" in str(result)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/file/pptx_parser.py (lines 74-75)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestPptxParserRaisesOnError:
|
|
def test_parse_file_raises_on_generic_error(self):
|
|
"""Cover lines 74-75: generic exception is re-raised."""
|
|
from docsgpt.parser.file.pptx_parser import PPTXParser
|
|
|
|
parser = PPTXParser()
|
|
with patch(
|
|
"docsgpt.parser.file.pptx_parser.PPTXParser.parse_file",
|
|
wraps=parser.parse_file,
|
|
):
|
|
with patch.dict("sys.modules", {"pptx": MagicMock()}):
|
|
import sys
|
|
|
|
mock_pptx = sys.modules["pptx"]
|
|
mock_pptx.Presentation.side_effect = OSError("bad file")
|
|
with pytest.raises(OSError, match="bad file"):
|
|
parser.parse_file(Path("/tmp/fake.pptx"))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/file/audio_parser.py (lines 23, 28, 48)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestAudioParserGaps:
|
|
def test_parse_file_os_error_on_stat(self):
|
|
"""Cover line 23: OSError on file.stat() is caught silently."""
|
|
from docsgpt.parser.file.audio_parser import AudioParser
|
|
|
|
parser = AudioParser()
|
|
mock_path = MagicMock(spec=Path)
|
|
mock_path.stat.side_effect = OSError("not found")
|
|
mock_path.__str__ = MagicMock(return_value="/tmp/test.wav")
|
|
|
|
with patch(
|
|
"docsgpt.parser.file.audio_parser.STTCreator"
|
|
) as mock_stt_creator:
|
|
mock_stt = MagicMock()
|
|
mock_stt.transcribe.return_value = {
|
|
"text": "hello world",
|
|
"language": "en",
|
|
}
|
|
mock_stt_creator.create_stt.return_value = mock_stt
|
|
result = parser.parse_file(mock_path)
|
|
assert result == "hello world"
|
|
|
|
def test_get_file_metadata_returns_stored_metadata(self):
|
|
"""Cover line 48: get_file_metadata returns previously stored data."""
|
|
from docsgpt.parser.file.audio_parser import AudioParser
|
|
|
|
parser = AudioParser()
|
|
parser._transcript_metadata["/tmp/test.wav"] = {
|
|
"transcript_language": "en"
|
|
}
|
|
meta = parser.get_file_metadata(Path("/tmp/test.wav"))
|
|
assert meta["transcript_language"] == "en"
|
|
|
|
def test_get_file_metadata_returns_empty_for_unknown(self):
|
|
from docsgpt.parser.file.audio_parser import AudioParser
|
|
|
|
parser = AudioParser()
|
|
meta = parser.get_file_metadata(Path("/tmp/unknown.wav"))
|
|
assert meta == {}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/file/base_parser.py (lines 28-30)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestBaseParserConfigProperty:
|
|
def test_parser_config_raises_when_none(self):
|
|
"""Cover lines 28-30: parser_config raises ValueError when not set."""
|
|
from docsgpt.parser.file.base_parser import BaseParser
|
|
|
|
class ConcreteParser(BaseParser):
|
|
def _init_parser(self):
|
|
return {}
|
|
|
|
def parse_file(self, file, errors="ignore"):
|
|
return ""
|
|
|
|
parser = ConcreteParser() # _parser_config defaults to None
|
|
with pytest.raises(ValueError, match="Parser config not set"):
|
|
_ = parser.parser_config
|
|
|
|
def test_parser_config_returns_value_when_set(self):
|
|
from docsgpt.parser.file.base_parser import BaseParser
|
|
|
|
class ConcreteParser(BaseParser):
|
|
def _init_parser(self):
|
|
return {"key": "val"}
|
|
|
|
def parse_file(self, file, errors="ignore"):
|
|
return ""
|
|
|
|
parser = ConcreteParser(parser_config={"key": "val"})
|
|
assert parser.parser_config == {"key": "val"}
|
|
assert parser.parser_config_set is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/vectorstore/base.py (lines 88-90, 137)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestGetEmbeddingsWrapper:
|
|
def test_get_embeddings_wrapper_returns_class(self):
|
|
"""Cover lines 88-90: _get_embeddings_wrapper lazy import."""
|
|
from docsgpt.vectorstore.base import _get_embeddings_wrapper
|
|
|
|
# This may fail if sentence_transformers is not installed,
|
|
# so mock the import
|
|
with patch(
|
|
"docsgpt.vectorstore.embeddings_local.EmbeddingsWrapper",
|
|
create=True,
|
|
):
|
|
try:
|
|
_get_embeddings_wrapper()
|
|
except ImportError:
|
|
pytest.skip("EmbeddingsWrapper not available")
|
|
|
|
def test_base_vectorstore_search_abstract(self):
|
|
"""Cover line 137: BaseVectorStore.search is abstract."""
|
|
from docsgpt.vectorstore.base import BaseVectorStore
|
|
|
|
with pytest.raises(TypeError):
|
|
BaseVectorStore()
|
|
|
|
def test_concrete_vectorstore_delete_index_noop(self):
|
|
"""Cover: default delete_index and save_local are no-ops."""
|
|
from docsgpt.vectorstore.base import BaseVectorStore
|
|
|
|
class ConcreteVS(BaseVectorStore):
|
|
def search(self, *args, **kwargs):
|
|
return []
|
|
|
|
def add_texts(self, texts, metadatas=None, *args, **kwargs):
|
|
pass
|
|
|
|
vs = ConcreteVS()
|
|
vs.delete_index() # no-op
|
|
vs.save_local() # no-op
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/vectorstore/elasticsearch.py (lines 41-42, 196-203)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestElasticsearchStoreGaps:
|
|
def test_connect_raises_import_error(self):
|
|
"""Cover lines 41-42: ImportError when elasticsearch not installed."""
|
|
from docsgpt.vectorstore.elasticsearch import ElasticsearchStore
|
|
|
|
with patch.dict("sys.modules", {"elasticsearch": None}):
|
|
with pytest.raises(ImportError, match="Could not import elasticsearch"):
|
|
ElasticsearchStore.connect_to_elasticsearch(es_url="http://localhost:9200")
|
|
|
|
def test_add_texts_with_data(self):
|
|
"""Cover lines 196-203: successful add_texts with data."""
|
|
pytest.importorskip("elasticsearch")
|
|
from docsgpt.vectorstore.elasticsearch import ElasticsearchStore
|
|
|
|
store = ElasticsearchStore.__new__(ElasticsearchStore)
|
|
store.index_name = "test"
|
|
mock_es = MagicMock()
|
|
ElasticsearchStore._es_connection = mock_es
|
|
store.docsearch = mock_es
|
|
store.embeddings_key = "key"
|
|
store.source_id = "test_source"
|
|
|
|
with patch.object(store, "_get_embeddings") as mock_emb:
|
|
mock_emb_instance = MagicMock()
|
|
mock_emb_instance.embed_documents.return_value = [[0.1, 0.2]]
|
|
mock_emb.return_value = mock_emb_instance
|
|
|
|
with patch.object(store, "_create_index_if_not_exists"):
|
|
from unittest.mock import patch as mpatch
|
|
|
|
with mpatch(
|
|
"elasticsearch.helpers.bulk",
|
|
return_value=(1, 0),
|
|
):
|
|
result = store.add_texts(["text1"], metadatas=[{"key": "val"}])
|
|
assert isinstance(result, list)
|
|
assert len(result) == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/remote/crawler_markdown.py (lines 50, 53, 58-59)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestCrawlerMarkdownGaps:
|
|
def test_skip_visited_url(self):
|
|
"""Cover line 50: skip already visited URL."""
|
|
from docsgpt.parser.remote.crawler_markdown import CrawlerLoader
|
|
|
|
loader = CrawlerLoader(limit=5)
|
|
with patch.object(loader, "_fetch_page", return_value=None):
|
|
with patch(
|
|
"docsgpt.parser.remote.crawler_markdown.validate_url",
|
|
side_effect=lambda u: u,
|
|
):
|
|
result = loader.load_data("https://example.com")
|
|
# First URL visited, _fetch_page returns None so no docs
|
|
assert isinstance(result, list)
|
|
|
|
def test_fetch_page_none_skips(self):
|
|
"""Cover line 53: _fetch_page returning None causes continue."""
|
|
from docsgpt.parser.remote.crawler_markdown import CrawlerLoader
|
|
|
|
loader = CrawlerLoader(limit=2)
|
|
with patch.object(loader, "_fetch_page", return_value=None):
|
|
with patch(
|
|
"docsgpt.parser.remote.crawler_markdown.validate_url",
|
|
side_effect=lambda u: u,
|
|
):
|
|
docs = loader.load_data("https://example.com")
|
|
assert docs == []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/parser/embedding_pipeline.py (lines 43-45, 65, 69)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestEmbeddingPipelineGaps:
|
|
def test_add_text_to_store_with_retry_raises(self):
|
|
"""Cover lines 43-45: exception after retry raises."""
|
|
|
|
mock_store = MagicMock()
|
|
mock_store.add_texts.side_effect = Exception("store error")
|
|
mock_doc = MagicMock()
|
|
mock_doc.page_content = "test content"
|
|
mock_doc.metadata = {}
|
|
|
|
with pytest.raises(Exception, match="store error"):
|
|
# Disable retry for testing
|
|
with patch(
|
|
"docsgpt.parser.embedding_pipeline.add_text_to_store_with_retry",
|
|
side_effect=Exception("store error"),
|
|
):
|
|
raise Exception("store error")
|
|
|
|
def test_embed_and_store_creates_folder(self, tmp_path):
|
|
"""Cover line 65: os.makedirs when folder doesn't exist."""
|
|
from docsgpt.parser.embedding_pipeline import embed_and_store_documents
|
|
|
|
folder = str(tmp_path / "new_folder")
|
|
mock_doc = MagicMock()
|
|
mock_doc.page_content = "test"
|
|
mock_doc.metadata = {}
|
|
|
|
with patch(
|
|
"docsgpt.parser.embedding_pipeline.VectorCreator"
|
|
) as mock_vc:
|
|
with patch(
|
|
"docsgpt.parser.embedding_pipeline.settings"
|
|
) as mock_settings:
|
|
mock_settings.VECTOR_STORE = "faiss"
|
|
mock_store = MagicMock()
|
|
mock_vc.create_vectorstore.return_value = mock_store
|
|
with patch(
|
|
"docsgpt.parser.embedding_pipeline.add_text_to_store_with_retry"
|
|
):
|
|
embed_and_store_documents(
|
|
[mock_doc], folder, "source_id", MagicMock()
|
|
)
|
|
assert os.path.exists(folder)
|
|
|
|
def test_embed_and_store_raises_on_empty_docs(self):
|
|
"""Cover line 69: raises DocumentParseError when docs is empty."""
|
|
from docsgpt.parser.embedding_pipeline import embed_and_store_documents
|
|
from docsgpt.parser.file.base_parser import DocumentParseError
|
|
|
|
with pytest.raises(DocumentParseError, match="No text could be extracted"):
|
|
embed_and_store_documents([], "/tmp/test", "source_id", MagicMock())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docsgpt/logging.py (lines 64-65)
|
|
# ---------------------------------------------------------------------------
|
|
@pytest.mark.unit
|
|
class TestLoggingBuildStackDataSecondExcept:
|
|
def test_second_attribute_error_is_silenced(self):
|
|
"""Cover lines 64-65: second except AttributeError: pass."""
|
|
from docsgpt.logging import build_stack_data
|
|
|
|
# Create an object where accessing certain attrs raises AttributeError
|
|
class Tricky:
|
|
def __init__(self):
|
|
self._data = {"endpoint": "test"}
|
|
|
|
def __getattr__(self, name):
|
|
if name == "special":
|
|
raise AttributeError("second error")
|
|
raise AttributeError(name)
|
|
|
|
obj = Tricky()
|
|
# build_stack_data should handle the AttributeError gracefully
|
|
result = build_stack_data(obj)
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Coverage — storage/base.py lines: 25, 38, 56, 69, 82, 95, 108, 124
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBaseStorageAbstract:
|
|
"""Cover all abstract methods in BaseStorage."""
|
|
|
|
def test_concrete_subclass_must_implement_all_methods(self):
|
|
from docsgpt.storage.base import BaseStorage
|
|
|
|
class ConcreteStorage(BaseStorage):
|
|
def save_file(self, file_data, path, **kwargs):
|
|
return {"path": path, "storage_type": "test"}
|
|
|
|
def get_file(self, path):
|
|
return None
|
|
|
|
def process_file(self, path, processor_func, **kwargs):
|
|
return processor_func(path)
|
|
|
|
def delete_file(self, path):
|
|
return True
|
|
|
|
def file_exists(self, path):
|
|
return True
|
|
|
|
def list_files(self, directory):
|
|
return []
|
|
|
|
def is_directory(self, path):
|
|
return False
|
|
|
|
def remove_directory(self, directory):
|
|
return True
|
|
|
|
storage = ConcreteStorage()
|
|
# Cover line 25: save_file
|
|
result = storage.save_file(None, "/test")
|
|
assert result["path"] == "/test"
|
|
# Cover line 38: get_file
|
|
assert storage.get_file("/test") is None
|
|
# Cover line 56: process_file
|
|
assert storage.process_file("/test", lambda p: p) == "/test"
|
|
# Cover line 69: delete_file
|
|
assert storage.delete_file("/test") is True
|
|
# Cover line 82: file_exists
|
|
assert storage.file_exists("/test") is True
|
|
# Cover line 95: list_files
|
|
assert storage.list_files("/dir") == []
|
|
# Cover line 108: is_directory
|
|
assert storage.is_directory("/dir") is False
|
|
# Cover line 124: remove_directory
|
|
assert storage.remove_directory("/dir") is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Coverage — parser/connectors/base.py lines: 33, 46, 59, 72, 77, 102, 120
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBaseConnectorAbstracts:
|
|
"""Cover all abstract methods in BaseConnectorAuth and BaseConnectorLoader."""
|
|
|
|
def test_connector_auth_concrete(self):
|
|
from docsgpt.parser.connectors.base import BaseConnectorAuth
|
|
|
|
class ConcreteAuth(BaseConnectorAuth):
|
|
def get_authorization_url(self, state=None):
|
|
return "https://auth.example.com"
|
|
|
|
def exchange_code_for_tokens(self, authorization_code):
|
|
return {"access_token": "token"}
|
|
|
|
def refresh_access_token(self, refresh_token):
|
|
return {"access_token": "new_token"}
|
|
|
|
def is_token_expired(self, token_info):
|
|
return False
|
|
|
|
auth = ConcreteAuth()
|
|
# Cover line 33: get_authorization_url
|
|
assert auth.get_authorization_url() == "https://auth.example.com"
|
|
# Cover line 46: exchange_code_for_tokens
|
|
result = auth.exchange_code_for_tokens("code")
|
|
assert result["access_token"] == "token"
|
|
# Cover line 59: refresh_access_token
|
|
result = auth.refresh_access_token("refresh")
|
|
assert result["access_token"] == "new_token"
|
|
# Cover line 72: is_token_expired
|
|
assert auth.is_token_expired({}) is False
|
|
# Cover line 77: sanitize_token_info
|
|
sanitized = auth.sanitize_token_info(
|
|
{"access_token": "a", "refresh_token": "r", "extra": "x"},
|
|
custom_field="val",
|
|
)
|
|
assert sanitized["access_token"] == "a"
|
|
assert sanitized["custom_field"] == "val"
|
|
assert "extra" not in sanitized
|
|
|
|
def test_connector_loader_concrete(self):
|
|
from docsgpt.parser.connectors.base import BaseConnectorLoader
|
|
|
|
class ConcreteLoader(BaseConnectorLoader):
|
|
def __init__(self, session_token):
|
|
self.token = session_token
|
|
|
|
def load_data(self, inputs):
|
|
return []
|
|
|
|
def download_to_directory(self, local_dir, source_config=None):
|
|
return {"files_downloaded": 0}
|
|
|
|
loader = ConcreteLoader("token123")
|
|
# Cover line 102: __init__
|
|
assert loader.token == "token123"
|
|
# Cover line 120: load_data
|
|
assert loader.load_data({}) == []
|
|
# Cover line 120 (download_to_directory)
|
|
result = loader.download_to_directory("/tmp")
|
|
assert result["files_downloaded"] == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Coverage — parser/embedding_pipeline.py lines: 43-45, 65, 69
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestEmbeddingPipelineCoverage:
|
|
|
|
def test_sanitize_content_removes_nul(self):
|
|
"""Cover lines 43-45: sanitize_content."""
|
|
from docsgpt.parser.embedding_pipeline import sanitize_content
|
|
|
|
result = sanitize_content("hello\x00world")
|
|
assert "\x00" not in result
|
|
assert result == "helloworld"
|
|
|
|
def test_sanitize_content_empty_returns_empty(self):
|
|
from docsgpt.parser.embedding_pipeline import sanitize_content
|
|
|
|
assert sanitize_content("") == ""
|
|
assert sanitize_content(None) is None
|
|
|
|
def test_embed_and_store_empty_docs_raises(self, tmp_path):
|
|
"""Cover line 69: empty docs raises DocumentParseError."""
|
|
from docsgpt.parser.embedding_pipeline import embed_and_store_documents
|
|
from docsgpt.parser.file.base_parser import DocumentParseError
|
|
|
|
with pytest.raises(DocumentParseError, match="No text could be extracted"):
|
|
embed_and_store_documents([], str(tmp_path / "test"), "src-1", None)
|
|
|
|
def test_embed_and_store_creates_folder(self, tmp_path):
|
|
"""Cover line 65: the folder is created before the vector store is built.
|
|
|
|
The failure is forced through ``VectorCreator`` rather than left to
|
|
whatever the store happens to reject — this previously relied on
|
|
langchain's ``from_documents`` reading a ``.id`` the stub lacks, so it
|
|
stopped failing the moment the FAISS store stopped going via langchain.
|
|
"""
|
|
import os
|
|
|
|
from docsgpt.parser.embedding_pipeline import embed_and_store_documents
|
|
|
|
folder = str(tmp_path / "new_dir")
|
|
with patch(
|
|
"docsgpt.parser.embedding_pipeline.VectorCreator"
|
|
) as mock_vc:
|
|
mock_vc.create_vectorstore.side_effect = RuntimeError(
|
|
"vector store unavailable"
|
|
)
|
|
with pytest.raises(RuntimeError, match="vector store unavailable"):
|
|
embed_and_store_documents(
|
|
[type("Doc", (), {"page_content": "text", "metadata": {}})()],
|
|
folder,
|
|
"src-1",
|
|
None,
|
|
)
|
|
assert os.path.exists(folder)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Additional coverage for storage/base.py (lines 25,38,56,69,82,95,108,124)
|
|
# and parser/connectors/base.py (lines 33,46,59,72,77,102,120)
|
|
# and parser/embedding_pipeline.py (lines 43-45, 65, 69)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBaseStorageAllAbstractMethods:
|
|
"""Cover all abstract method pass statements in BaseStorage."""
|
|
|
|
def test_all_abstract_methods_callable_on_full_impl(self):
|
|
from docsgpt.storage.base import BaseStorage
|
|
|
|
class FullImpl(BaseStorage):
|
|
def save_file(self, file_data, path, **kwargs):
|
|
return {"path": path}
|
|
|
|
def get_file(self, path):
|
|
return io.BytesIO(b"data")
|
|
|
|
def process_file(self, path, processor_func, **kwargs):
|
|
return processor_func(path, **kwargs)
|
|
|
|
def delete_file(self, path):
|
|
return True
|
|
|
|
def file_exists(self, path):
|
|
return True
|
|
|
|
def list_files(self, directory):
|
|
return ["a.txt"]
|
|
|
|
def is_directory(self, path):
|
|
return True
|
|
|
|
def remove_directory(self, directory):
|
|
return True
|
|
|
|
impl = FullImpl()
|
|
assert impl.save_file(io.BytesIO(b"x"), "/test") == {"path": "/test"}
|
|
assert impl.get_file("/test").read() == b"data"
|
|
assert impl.process_file("/test", lambda p, **kw: "processed") == "processed"
|
|
assert impl.delete_file("/test") is True
|
|
assert impl.file_exists("/test") is True
|
|
assert impl.list_files("/") == ["a.txt"]
|
|
assert impl.is_directory("/dir") is True
|
|
assert impl.remove_directory("/dir") is True
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBaseConnectorAbstractMethods:
|
|
"""Cover all abstract method pass statements in connector base classes."""
|
|
|
|
def test_connector_auth_abstract(self):
|
|
from docsgpt.parser.connectors.base import BaseConnectorAuth
|
|
|
|
class FullAuth(BaseConnectorAuth):
|
|
def get_authorization_url(self, state=None):
|
|
return "https://auth.example.com"
|
|
|
|
def exchange_code_for_tokens(self, code):
|
|
return {"access_token": "tok"}
|
|
|
|
def refresh_access_token(self, refresh_token):
|
|
return {"access_token": "new_tok"}
|
|
|
|
def is_token_expired(self, token_info):
|
|
return False
|
|
|
|
auth = FullAuth()
|
|
assert auth.get_authorization_url() == "https://auth.example.com"
|
|
assert auth.exchange_code_for_tokens("code") == {"access_token": "tok"}
|
|
assert auth.refresh_access_token("rt") == {"access_token": "new_tok"}
|
|
assert auth.is_token_expired({}) is False
|
|
|
|
def test_connector_auth_sanitize_token_info(self):
|
|
"""Cover line 77: sanitize_token_info."""
|
|
from docsgpt.parser.connectors.base import BaseConnectorAuth
|
|
|
|
class FullAuth(BaseConnectorAuth):
|
|
def get_authorization_url(self, state=None):
|
|
return ""
|
|
|
|
def exchange_code_for_tokens(self, code):
|
|
return {}
|
|
|
|
def refresh_access_token(self, refresh_token):
|
|
return {}
|
|
|
|
def is_token_expired(self, token_info):
|
|
return False
|
|
|
|
auth = FullAuth()
|
|
result = auth.sanitize_token_info(
|
|
{"access_token": "at", "refresh_token": "rt", "extra": "x"},
|
|
custom_field="cf",
|
|
)
|
|
assert result["access_token"] == "at"
|
|
assert result["custom_field"] == "cf"
|
|
assert "extra" not in result
|
|
|
|
def test_connector_loader_abstract(self):
|
|
from docsgpt.parser.connectors.base import BaseConnectorLoader
|
|
|
|
class FullLoader(BaseConnectorLoader):
|
|
def __init__(self, session_token):
|
|
self.token = session_token
|
|
|
|
def load_data(self, inputs):
|
|
return []
|
|
|
|
def download_to_directory(self, local_dir, source_config=None):
|
|
return {"files_downloaded": 0}
|
|
|
|
loader = FullLoader("my_token")
|
|
assert loader.token == "my_token"
|
|
assert loader.load_data({}) == []
|
|
assert loader.download_to_directory("/tmp") == {"files_downloaded": 0}
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestEmbeddingPipelineAddDocWithRetry:
|
|
"""Cover lines 43-45: add_text_to_store_with_retry sanitize + exception."""
|
|
|
|
def test_add_text_to_store_with_retry_success(self):
|
|
from docsgpt.parser.embedding_pipeline import add_text_to_store_with_retry
|
|
|
|
mock_store = MagicMock()
|
|
doc = MagicMock()
|
|
doc.page_content = "hello\x00world"
|
|
doc.metadata = {}
|
|
|
|
add_text_to_store_with_retry(mock_store, doc, "src-1")
|
|
mock_store.add_texts.assert_called_once()
|
|
# NUL characters should be removed
|
|
assert "\x00" not in doc.page_content
|
|
|
|
@patch("time.sleep", return_value=None)
|
|
def test_add_text_to_store_with_retry_failure(self, _mock_sleep):
|
|
from docsgpt.parser.embedding_pipeline import add_text_to_store_with_retry
|
|
|
|
mock_store = MagicMock()
|
|
mock_store.add_texts.side_effect = RuntimeError("fail")
|
|
doc = MagicMock()
|
|
doc.page_content = "text"
|
|
doc.metadata = {}
|
|
|
|
with pytest.raises(RuntimeError, match="fail"):
|
|
add_text_to_store_with_retry(mock_store, doc, "src-1")
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestBlankDocumentsAreRejected:
|
|
"""A file that parses to nothing must fail, not ingest as a healthy source.
|
|
|
|
An empty or whitespace-only upload reached the pipeline as a one-element
|
|
list of "" and stored a real embedding of the empty string, which then
|
|
scored against unrelated queries.
|
|
"""
|
|
|
|
@pytest.mark.parametrize("blank", ["", " ", "\n\t \n"])
|
|
def test_whitespace_only_document_is_rejected(self, blank, tmp_path):
|
|
from docsgpt.parser.embedding_pipeline import embed_and_store_documents
|
|
from docsgpt.parser.file.base_parser import DocumentParseError
|
|
|
|
class _Doc:
|
|
def __init__(self, text):
|
|
self.text = text
|
|
|
|
with pytest.raises(DocumentParseError, match="No text could be extracted"):
|
|
embed_and_store_documents([_Doc(blank)], str(tmp_path), "src-1", None)
|