mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-04 04:12:36 +00:00
Alex
f6400cd736
Introduces a per-source config contract that makes RAG behavior strategy-dispatched instead of a single hardcoded path. Every source gains a validated JSONB config; an empty/absent config reproduces current behavior byte-for-byte, and the whole path is gated by PER_SOURCE_RETRIEVAL_ENABLED. Foundation: sources.config JSONB column + migration 0022_source_config; SourceConfig/ChunkingConfig/RetrievalConfig pydantic models (strict on write, lenient on read); ChunkerCreator and RetrieverCreator.register registries; config threaded through the upload routes, ingest/remote/connector workers, and reingest. Retrieval: a Dispatcher groups sources by retriever key (all-classic collapses to today's single ClassicRAG under one shared token budget; non-classic retrievers get their own instance), removing the previous single-global-retriever collapse in stream_processor. Per-source chunks, score_threshold (honored for pgvector/mongodb, safely ignored elsewhere), and rephrase_query toggle. New PATCH /api/sources/<id>/config with team-aware (effective_write_owner) authz and a requires_reingest signal. Chunking strategies: recursive, markdown, parent_child (selectable per source; re-ingest to apply). Search exposure: per-source prefetch vs agentic_tool for agentic/research agents. Map-reduce prescreen: optional LLM relevance pre-filter implemented as a composable post-retrieval stage that wraps any retriever. Backend and frontend (shared Retrieval options panel + edit modal) with tests; backend suite and frontend vitest green. Excludes the wiki and GraphRAG flagships.
74 lines
2.1 KiB
Python
74 lines
2.1 KiB
Python
"""Migration round-trip test for 0022_source_config."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _alembic_ini() -> Path:
|
|
return Path(__file__).resolve().parents[3] / "application" / "alembic.ini"
|
|
|
|
|
|
def _run_alembic(url: str, *args: str) -> None:
|
|
subprocess.check_call(
|
|
[sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), *args],
|
|
timeout=60,
|
|
env={**os.environ, "POSTGRES_URI": url},
|
|
)
|
|
|
|
|
|
def _alembic_version(conn) -> str:
|
|
return conn.execute(text("SELECT version_num FROM alembic_version")).scalar()
|
|
|
|
|
|
def _column_exists(conn, table: str, column: str) -> bool:
|
|
row = conn.execute(
|
|
text(
|
|
"SELECT 1 FROM information_schema.columns "
|
|
"WHERE table_name = :t AND column_name = :c"
|
|
),
|
|
{"t": table, "c": column},
|
|
).fetchone()
|
|
return row is not None
|
|
|
|
|
|
_0022 = "0022_source_config"
|
|
_0021 = "0021_teams"
|
|
|
|
|
|
class TestMigration0022RoundTrip:
|
|
def test_head_has_config_column(self, pg_engine):
|
|
with pg_engine.connect() as conn:
|
|
assert _alembic_version(conn) >= _0022
|
|
assert _column_exists(conn, "sources", "config")
|
|
|
|
def test_config_server_default_backfills_empty(self, pg_engine):
|
|
with pg_engine.begin() as conn:
|
|
row = conn.execute(
|
|
text(
|
|
"INSERT INTO sources (user_id, name) "
|
|
"VALUES ('u-cfg', 's') RETURNING config"
|
|
)
|
|
).fetchone()
|
|
assert row[0] == {}
|
|
|
|
def test_downgrade_drops_then_upgrade_restores(self, pg_engine):
|
|
url = pg_engine.url.render_as_string(hide_password=False)
|
|
_run_alembic(url, "downgrade", _0021)
|
|
with pg_engine.connect() as conn:
|
|
assert _alembic_version(conn) == _0021
|
|
assert not _column_exists(conn, "sources", "config")
|
|
_run_alembic(url, "upgrade", "head")
|
|
with pg_engine.connect() as conn:
|
|
assert _alembic_version(conn) >= _0022
|
|
assert _column_exists(conn, "sources", "config")
|