mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-04 06:12:49 +00:00
Alex
f6400cd736
Introduces a per-source config contract that makes RAG behavior strategy-dispatched instead of a single hardcoded path. Every source gains a validated JSONB config; an empty/absent config reproduces current behavior byte-for-byte, and the whole path is gated by PER_SOURCE_RETRIEVAL_ENABLED. Foundation: sources.config JSONB column + migration 0022_source_config; SourceConfig/ChunkingConfig/RetrievalConfig pydantic models (strict on write, lenient on read); ChunkerCreator and RetrieverCreator.register registries; config threaded through the upload routes, ingest/remote/connector workers, and reingest. Retrieval: a Dispatcher groups sources by retriever key (all-classic collapses to today's single ClassicRAG under one shared token budget; non-classic retrievers get their own instance), removing the previous single-global-retriever collapse in stream_processor. Per-source chunks, score_threshold (honored for pgvector/mongodb, safely ignored elsewhere), and rephrase_query toggle. New PATCH /api/sources/<id>/config with team-aware (effective_write_owner) authz and a requires_reingest signal. Chunking strategies: recursive, markdown, parent_child (selectable per source; re-ingest to apply). Search exposure: per-source prefetch vs agentic_tool for agentic/research agents. Map-reduce prescreen: optional LLM relevance pre-filter implemented as a composable post-retrieval stage that wraps any retriever. Backend and frontend (shared Retrieval options panel + edit modal) with tests; backend suite and frontend vitest green. Excludes the wiki and GraphRAG flagships.
76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
"""Tests for ChunkerCreator registry + byte-identical classic_chunk parity."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from application.parser.chunking import Chunker
|
|
from application.parser.chunking_creator import ChunkerCreator
|
|
from application.parser.schema.base import Document
|
|
|
|
|
|
def _docs():
|
|
"""Representative mix: a small doc, a large splittable doc, a tiny doc."""
|
|
return [
|
|
Document(text="A short paragraph of text.", doc_id="small"),
|
|
Document(text="word " * 4000, doc_id="large"),
|
|
Document(text="tiny", doc_id="below-min"),
|
|
Document(text="header1\nheader2\nheader3\n" + "lorem " * 3000, doc_id="hdr"),
|
|
]
|
|
|
|
|
|
def _serialize(chunks):
|
|
"""Stable, comparable view of a chunk list."""
|
|
return [(c.doc_id, c.text, c.extra_info) for c in chunks]
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestChunkerCreator:
|
|
def test_classic_chunk_is_registered(self):
|
|
assert "classic_chunk" in ChunkerCreator.chunkers
|
|
assert ChunkerCreator.chunkers["classic_chunk"] is Chunker
|
|
|
|
def test_create_chunker_returns_chunker(self):
|
|
chunker = ChunkerCreator.create_chunker(
|
|
"classic_chunk", max_tokens=1250, min_tokens=150,
|
|
)
|
|
assert isinstance(chunker, Chunker)
|
|
assert chunker.max_tokens == 1250
|
|
assert chunker.min_tokens == 150
|
|
|
|
def test_create_chunker_unknown_strategy_raises(self):
|
|
with pytest.raises(ValueError, match="No chunker class found"):
|
|
ChunkerCreator.create_chunker("does_not_exist")
|
|
|
|
def test_register_is_idempotent_and_overrides(self):
|
|
class Dummy:
|
|
def __init__(self, **kwargs):
|
|
self.kwargs = kwargs
|
|
|
|
try:
|
|
ChunkerCreator.register("dummy_strategy", Dummy)
|
|
inst = ChunkerCreator.create_chunker("dummy_strategy", x=1)
|
|
assert isinstance(inst, Dummy)
|
|
assert inst.kwargs == {"x": 1}
|
|
finally:
|
|
ChunkerCreator.chunkers.pop("dummy_strategy", None)
|
|
|
|
|
|
@pytest.mark.unit
|
|
class TestByteIdenticalParity:
|
|
def test_creator_output_matches_direct_chunker(self):
|
|
# The refactor must not change classic_chunk output: a chunker built
|
|
# via ChunkerCreator must produce identical chunks to the old direct
|
|
# ``Chunker(...)`` instantiation for the same input + params.
|
|
params = dict(max_tokens=1250, min_tokens=150, duplicate_headers=False)
|
|
|
|
direct = Chunker(chunking_strategy="classic_chunk", **params)
|
|
via_creator = ChunkerCreator.create_chunker(
|
|
"classic_chunk", chunking_strategy="classic_chunk", **params,
|
|
)
|
|
|
|
direct_out = direct.chunk(documents=_docs())
|
|
creator_out = via_creator.chunk(documents=_docs())
|
|
|
|
assert _serialize(creator_out) == _serialize(direct_out)
|