diff --git a/docs/content/Deploying/Settings-Reference.mdx b/docs/content/Deploying/Settings-Reference.mdx index 02876714..30221972 100644 --- a/docs/content/Deploying/Settings-Reference.mdx +++ b/docs/content/Deploying/Settings-Reference.mdx @@ -271,7 +271,7 @@ Context window assumed when the model is not found in the registry. ### `RESERVED_TOKENS` -Type `dict`, default `{"system_prompt": 500, "current_query": 500, "safety_buffer": 1000}`. +Type `dict[str, int]`, default `{"system_prompt": 500, "current_query": 500, "safety_buffer": 1000}`. Tokens held back from the context window for the system prompt, the query and a safety buffer. @@ -378,7 +378,7 @@ Where embedding models and their tokenizers are cached. Persistent by default: F ### `EMBEDDINGS_POOLING` -Type `str`, default unset. +Type `"cls" | "mean"`, default unset. Pooling strategy ("cls" or "mean"). Read from the model's own repository; set only for a repository that declares none, or to override what it declares. @@ -402,7 +402,7 @@ Celery queue the embed task is routed to. ### `EMBEDDINGS_DELEGATE_TIMEOUT` -Type `int`, default `60`. +Type `int`, default `60`, must be > 0. Seconds the API waits for the worker to return an embedding. @@ -443,9 +443,9 @@ Model for ingest-time graph extraction; unset reuses LLM_PROVIDER/LLM_NAME. ### `GRAPHRAG_MAX_CHUNKS_FOR_EXTRACTION` -Type `int`, default `2000`. +Type `int`, default `2000`, must be >= 0. -Hard cap on chunks extracted per source (cost control). +Hard cap on chunks extracted per source (cost control); 0 extracts nothing. ## Vector stores @@ -668,7 +668,7 @@ Tasks prefetched per worker process; 1 caps SIGKILL loss to one task. ### `CELERY_VISIBILITY_TIMEOUT` -Type `int`, default `3600`. +Type `int`, default `3600`, must be > 0. Broker visibility timeout in seconds. Must exceed the longest legitimate task runtime but stay short enough that SIGKILLed tasks redeliver promptly. @@ -1220,13 +1220,13 @@ Length of the replay budget window. ### `MESSAGE_EVENTS_RETENTION_DAYS` -Type `int`, default `14`. +Type `int`, default `14`, must be > 0. Retention for the message_events journal, enforced by the cleanup_message_events beat task. Replay only needs streams a client could still be tailing. ### `REMOTE_DEVICE_SESSION_IDLE_SECONDS` -Type `int`, default `60`. +Type `int`, default `60`, must be > 0. Seconds without a heartbeat before a remote-device session is considered idle. @@ -1238,19 +1238,19 @@ Require signed commands from remote devices. ### `REMOTE_DEVICE_PAIRING_TTL_SECONDS` -Type `int`, default `600`. +Type `int`, default `600`, must be > 0. Lifetime of a pairing code. ### `REMOTE_DEVICE_CMD_QUEUE_TTL_SECONDS` -Type `int`, default `900`. +Type `int`, default `900`, must be > 605. Redis TTL of the per-device command queue, routing invocations cross-process so a scheduled run reaches the web-held device session. Must exceed the max drain deadline (605s) so a command for a briefly-offline device isn't evicted before its own drain gives up. ### `REMOTE_DEVICE_INVOCATION_TTL_SECONDS` -Type `int`, default `900`. +Type `int`, default `900`, must be > 0. Redis TTL of a pending remote-device invocation. @@ -1273,13 +1273,13 @@ Default agent type for agentless chats. ### `DEFAULT_AGENT_LIMITS` -Type `dict`, default `{"token_limit": 50000, "request_limit": 500}`. +Type `dict[str, int]`, default `{"token_limit": 50000, "request_limit": 500}`. Per-agent default quotas: tokens and requests. ### `DEFAULT_CHAT_TOOLS` -Type `list`, default `["memory", "read_webpage", "scheduler"]`. +Type `list[str]`, default `["memory", "read_webpage", "scheduler"]`. Config-free tools on by default in agentless chats. scheduler is dual-registered in BUILTIN_AGENT_TOOLS so one synthetic id resolves via defaults or the agent picker. Add code_executor and artifact_generator once a sandbox runner is configured; both execute through it and would fail on every call without one. @@ -1386,13 +1386,13 @@ Master switch; False disables every stage. ### `GUARDRAILS_CHECKS_ENABLED` -Type `list`, default `[]`. +Type `list[str]`, default `[]`. Allowlist of GuardrailCreator.checks keys; empty means every registered check. ### `GUARDRAILS_FLOOR` -Type `dict`, default `{}`. +Type `dict[str, Any]`, default `{}`. A GuardrailsConfig fragment every agent inherits and cannot weaken; agents may add controls or make an action stricter, never looser. "enabled" is required; without it the floor parses but applies to nothing. Example: \{"enabled": true, "mode": "scan_all", "controls": [\{"check": "secrets", "stage": "output", "action": "redact"\}]\} @@ -1463,7 +1463,7 @@ How far ahead a one-off run may be scheduled, in seconds (one year). ### `SCHEDULE_RUN_OUTPUT_RETENTION_DAYS` -Type `int`, default `90`. +Type `int`, default `90`, must be > 0. Days scheduled-run output is kept. diff --git a/docsgpt/core/settings/_shared.py b/docsgpt/core/settings/_shared.py index 235ee07b..59723bbf 100644 --- a/docsgpt/core/settings/_shared.py +++ b/docsgpt/core/settings/_shared.py @@ -9,16 +9,43 @@ domain's definitions live in their own module. from __future__ import annotations +import types +import typing from typing import Any, Optional +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict +def _is_optional_str(annotation: Any) -> bool: + if typing.get_origin(annotation) not in (typing.Union, types.UnionType): + return False + return set(typing.get_args(annotation)) == {str, type(None)} + + class SettingsGroup(BaseSettings): - """Base for one domain's settings; groups are composed into ``Settings``.""" + """Base for one domain's settings; groups are composed into ``Settings``. + + Every ``Optional[str]`` field treats the spellings an unset value has in a + ``.env`` file (``KEY=``, ``KEY=None``, whitespace) as ``None``, so a check + like ``if settings.OIDC_ISSUER`` or a fallback like ``settings.X or default`` + sees "unset" rather than a truthy placeholder string. Real values are + stripped. Fields typed ``str`` keep whatever they are given. + """ model_config = SettingsConfigDict(extra="ignore") + @model_validator(mode="before") + @classmethod + def _unset_optional_strings(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + data = dict(data) + for name, field in cls.model_fields.items(): + if name in data and _is_optional_str(field.annotation): + data[name] = normalize_secret(data[name]) + return data + def normalize_choice(value: Any) -> Any: """Case-fold a closed-choice setting so ``PGVector`` and ``pgvector`` are the same choice.""" diff --git a/docsgpt/core/settings/auth.py b/docsgpt/core/settings/auth.py index 04b039bc..c9309775 100644 --- a/docsgpt/core/settings/auth.py +++ b/docsgpt/core/settings/auth.py @@ -85,11 +85,6 @@ class AuthSettings(SettingsGroup): default=None, description="Bearer token for IdP SCIM clients (required when SCIM is enabled)." ) - @field_validator("INTERNAL_KEY", mode="before") - @classmethod - def _normalize_auth_secrets(cls, v): - return normalize_secret(v) - @field_validator("AUTH_TYPE", mode="before") @classmethod def _normalize_auth_type(cls, v): diff --git a/docsgpt/core/settings/embeddings.py b/docsgpt/core/settings/embeddings.py index 3ac749aa..f8c5c130 100644 --- a/docsgpt/core/settings/embeddings.py +++ b/docsgpt/core/settings/embeddings.py @@ -2,12 +2,12 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional from pydantic import Field, field_validator from docsgpt.core.paths import home_dir -from docsgpt.core.settings._shared import SettingsGroup, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class EmbeddingsSettings(SettingsGroup): @@ -56,7 +56,7 @@ class EmbeddingsSettings(SettingsGroup): "default is the temp dir." ), ) - EMBEDDINGS_POOLING: Optional[str] = Field( + EMBEDDINGS_POOLING: Optional[Literal["cls", "mean"]] = Field( default=None, description=( 'Pooling strategy ("cls" or "mean"). Read from the model\'s own repository; set only for a ' @@ -79,10 +79,10 @@ class EmbeddingsSettings(SettingsGroup): ) EMBEDDINGS_QUEUE: str = Field(default="embeddings", description="Celery queue the embed task is routed to.") EMBEDDINGS_DELEGATE_TIMEOUT: int = Field( - default=60, description="Seconds the API waits for the worker to return an embedding." + default=60, gt=0, description="Seconds the API waits for the worker to return an embedding." ) - @field_validator("EMBEDDINGS_KEY", mode="before") + @field_validator("EMBEDDINGS_POOLING", mode="before") @classmethod - def _normalize_embeddings_secrets(cls, v): - return normalize_secret(v) + def _normalize_pooling(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/events.py b/docsgpt/core/settings/events.py index f8ca6742..3b170be9 100644 --- a/docsgpt/core/settings/events.py +++ b/docsgpt/core/settings/events.py @@ -58,6 +58,7 @@ class EventsSettings(SettingsGroup): EVENTS_REPLAY_BUDGET_WINDOW_SECONDS: int = Field(default=60, description="Length of the replay budget window.") MESSAGE_EVENTS_RETENTION_DAYS: int = Field( default=14, + gt=0, description=( "Retention for the message_events journal, enforced by the cleanup_message_events beat task. Replay " "only needs streams a client could still be tailing." @@ -66,14 +67,15 @@ class EventsSettings(SettingsGroup): # Remote Device feature. REMOTE_DEVICE_SESSION_IDLE_SECONDS: int = Field( - default=60, description="Seconds without a heartbeat before a remote-device session is considered idle." + default=60, gt=0, description="Seconds without a heartbeat before a remote-device session is considered idle." ) REMOTE_DEVICE_REQUIRE_SIGNATURE: bool = Field( default=False, description="Require signed commands from remote devices." ) - REMOTE_DEVICE_PAIRING_TTL_SECONDS: int = Field(default=600, description="Lifetime of a pairing code.") + REMOTE_DEVICE_PAIRING_TTL_SECONDS: int = Field(default=600, gt=0, description="Lifetime of a pairing code.") REMOTE_DEVICE_CMD_QUEUE_TTL_SECONDS: int = Field( default=900, + gt=605, description=( "Redis TTL of the per-device command queue, routing invocations cross-process so a scheduled run " "reaches the web-held device session. Must exceed the max drain deadline (605s) so a command for a " @@ -81,7 +83,7 @@ class EventsSettings(SettingsGroup): ), ) REMOTE_DEVICE_INVOCATION_TTL_SECONDS: int = Field( - default=900, description="Redis TTL of a pending remote-device invocation." + default=900, gt=0, description="Redis TTL of a pending remote-device invocation." ) REMOTE_DEVICE_OUTPUT_STREAM_MAXLEN: int = Field( default=10_000, description="Cap on buffered output entries per remote-device invocation stream." diff --git a/docsgpt/core/settings/llm.py b/docsgpt/core/settings/llm.py index 11c17cac..173c864a 100644 --- a/docsgpt/core/settings/llm.py +++ b/docsgpt/core/settings/llm.py @@ -5,10 +5,10 @@ from __future__ import annotations import os from typing import Optional -from pydantic import Field, field_validator +from pydantic import Field from docsgpt.core.paths import home_dir -from docsgpt.core.settings._shared import SettingsGroup, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup class LLMSettings(SettingsGroup): @@ -104,18 +104,3 @@ class LLMSettings(SettingsGroup): OPENAI_REASONING_SUMMARY: str = Field( default="auto", description="Reasoning summary mode requested from the Responses API." ) - - @field_validator( - "API_KEY", - "OPENAI_API_KEY", - "ANTHROPIC_API_KEY", - "GOOGLE_API_KEY", - "GROQ_API_KEY", - "HUGGINGFACE_API_KEY", - "NOVITA_API_KEY", - "FALLBACK_LLM_API_KEY", - mode="before", - ) - @classmethod - def _normalize_llm_secrets(cls, v): - return normalize_secret(v) diff --git a/docsgpt/core/settings/reference.py b/docsgpt/core/settings/reference.py index 02d3721d..e5aa3392 100644 --- a/docsgpt/core/settings/reference.py +++ b/docsgpt/core/settings/reference.py @@ -62,7 +62,9 @@ def _type_name(annotation: Any) -> str: if origin is Literal: return " | ".join(json.dumps(v) for v in typing.get_args(annotation)) if origin is not None: - return getattr(origin, "__name__", str(origin)) + name = getattr(origin, "__name__", str(origin)) + args = typing.get_args(annotation) + return f"{name}[{', '.join(_type_name(arg) for arg in args)}]" if args else name return getattr(annotation, "__name__", str(annotation)) diff --git a/docsgpt/core/settings/retrieval.py b/docsgpt/core/settings/retrieval.py index 8c716646..651d3f3d 100644 --- a/docsgpt/core/settings/retrieval.py +++ b/docsgpt/core/settings/retrieval.py @@ -29,7 +29,7 @@ class RetrievalSettings(SettingsGroup): default=None, description="Model for ingest-time graph extraction; unset reuses LLM_PROVIDER/LLM_NAME." ) GRAPHRAG_MAX_CHUNKS_FOR_EXTRACTION: int = Field( - default=2000, description="Hard cap on chunks extracted per source (cost control)." + default=2000, ge=0, description="Hard cap on chunks extracted per source (cost control); 0 extracts nothing." ) @field_validator("VECTOR_STORE", mode="before") diff --git a/docsgpt/core/settings/scheduler.py b/docsgpt/core/settings/scheduler.py index 00a9f455..b86d9995 100644 --- a/docsgpt/core/settings/scheduler.py +++ b/docsgpt/core/settings/scheduler.py @@ -25,4 +25,4 @@ class SchedulerSettings(SettingsGroup): SCHEDULE_ONCE_MAX_HORIZON: int = Field( default=31_536_000, description="How far ahead a one-off run may be scheduled, in seconds (one year)." ) - SCHEDULE_RUN_OUTPUT_RETENTION_DAYS: int = Field(default=90, description="Days scheduled-run output is kept.") + SCHEDULE_RUN_OUTPUT_RETENTION_DAYS: int = Field(default=90, gt=0, description="Days scheduled-run output is kept.") diff --git a/docsgpt/core/settings/speech.py b/docsgpt/core/settings/speech.py index da5c245e..f25380b0 100644 --- a/docsgpt/core/settings/speech.py +++ b/docsgpt/core/settings/speech.py @@ -6,7 +6,7 @@ from typing import Literal, Optional from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup, normalize_choice, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class SpeechSettings(SettingsGroup): @@ -25,11 +25,6 @@ class SpeechSettings(SettingsGroup): STT_ENABLE_TIMESTAMPS: bool = Field(default=False, description="Return word/segment timestamps.") STT_ENABLE_DIARIZATION: bool = Field(default=False, description="Label speakers in the transcript.") - @field_validator("ELEVENLABS_API_KEY", mode="before") - @classmethod - def _normalize_speech_secrets(cls, v): - return normalize_secret(v) - @field_validator("TTS_PROVIDER", "STT_PROVIDER", mode="before") @classmethod def _normalize_speech_providers(cls, v): diff --git a/docsgpt/core/settings/vectorstores.py b/docsgpt/core/settings/vectorstores.py index 067d1df0..010eefa3 100644 --- a/docsgpt/core/settings/vectorstores.py +++ b/docsgpt/core/settings/vectorstores.py @@ -8,7 +8,7 @@ from pydantic import Field, field_validator from docsgpt.core.db_uri import normalize_pgvector_connection_string from docsgpt.core.paths import home_dir -from docsgpt.core.settings._shared import SettingsGroup, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup class VectorStoreSettings(SettingsGroup): @@ -82,8 +82,3 @@ class VectorStoreSettings(SettingsGroup): @classmethod def _normalize_pgvector_connection_string(cls, v): return normalize_pgvector_connection_string(v) - - @field_validator("QDRANT_API_KEY", mode="before") - @classmethod - def _normalize_vectorstore_secrets(cls, v): - return normalize_secret(v) diff --git a/docsgpt/core/settings/workers.py b/docsgpt/core/settings/workers.py index 86246cab..96efc8f2 100644 --- a/docsgpt/core/settings/workers.py +++ b/docsgpt/core/settings/workers.py @@ -17,6 +17,7 @@ class WorkerSettings(SettingsGroup): ) CELERY_VISIBILITY_TIMEOUT: int = Field( default=3600, + gt=0, description=( "Broker visibility timeout in seconds. Must exceed the longest legitimate task runtime but stay " "short enough that SIGKILLed tasks redeliver promptly." diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index df0e0e07..db78a45b 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -6,6 +6,8 @@ validators from every group applied) and that the generated reference page tracks the definitions. """ +import types +import typing import warnings from pathlib import Path @@ -23,11 +25,22 @@ SECRET_FIELDS = ( "GROQ_API_KEY", "HUGGINGFACE_API_KEY", "NOVITA_API_KEY", + "OPEN_ROUTER_API_KEY", "EMBEDDINGS_KEY", "FALLBACK_LLM_API_KEY", "QDRANT_API_KEY", + "ELASTIC_PASSWORD", "ELEVENLABS_API_KEY", "INTERNAL_KEY", + "SCIM_TOKEN", + "OIDC_ISSUER", + "GITHUB_ACCESS_TOKEN", + "MICROSOFT_AUTHORITY", + "MCP_OAUTH_REDIRECT_URI", + "S3_ACCESS_KEY_ID", + "S3_SECRET_ACCESS_KEY", + "SANDBOX_GATEWAY_AUTH_TOKEN", + "DAYTONA_API_KEY", ) @@ -69,10 +82,27 @@ class TestComposition: class TestValidators: """Validators live on the group that owns the field; composition must keep all of them. - Two groups defining a validator under the same method name would silently - keep only one, so this checks every secret field, across every group. + Pydantic collects validators by method name across the MRO, so two groups + defining one under the same name would silently keep only one; the checks + below span every group. """ + def test_every_optional_string_treats_unset_spellings_as_none(self): + names = [ + name + for name, field in Settings.model_fields.items() + if typing.get_origin(field.annotation) in (typing.Union, types.UnionType) + and set(typing.get_args(field.annotation)) == {str, type(None)} + ] + assert len(names) > 60 + loaded = Settings.model_validate({name: " None " for name in names}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert [name for name in names if getattr(loaded, name) is not None] == [] + + def test_plain_strings_keep_empty_values(self): + assert Settings.model_validate({"MILVUS_TOKEN": "", "JWT_SECRET_KEY": ""}).MILVUS_TOKEN == "" + @pytest.mark.parametrize("name", SECRET_FIELDS) @pytest.mark.parametrize("raw", ["None", "none", "", " "]) def test_unset_secret_spellings_become_none(self, name, raw): @@ -125,6 +155,11 @@ class TestCrossFieldRules: with pytest.raises(ValidationError, match="AUTH_TYPE=oidc requires settings: OIDC_CLIENT_ID, OIDC_FRONTEND_URL"): Settings.model_validate({"AUTH_TYPE": "oidc", "OIDC_ISSUER": self.OIDC["OIDC_ISSUER"]}) + @pytest.mark.parametrize("raw", ["", "None", " "]) + def test_oidc_unset_spellings_do_not_satisfy_the_requirement(self, raw): + with pytest.raises(ValidationError, match="OIDC_CLIENT_ID"): + Settings.model_validate({"AUTH_TYPE": "oidc", **self.OIDC, "OIDC_CLIENT_ID": raw}) + def test_oidc_with_required_settings_loads(self): assert Settings.model_validate({"AUTH_TYPE": "OIDC", **self.OIDC}).AUTH_TYPE == "oidc" @@ -154,6 +189,7 @@ class TestClosedChoices: ("TTS_PROVIDER", "ElevenLabs", "elevenlabs"), ("STT_PROVIDER", "", "none"), ("TTS_PROVIDER", "NONE", "none"), + ("EMBEDDINGS_POOLING", "CLS", "cls"), ], ) def test_choices_are_case_insensitive(self, name, raw, expected): @@ -169,6 +205,7 @@ class TestClosedChoices: ("SANDBOX_BACKEND", "docker"), ("DOC_PARSER_ENGINE", "fast"), ("STT_PROVIDER", "whisper"), + ("EMBEDDINGS_POOLING", "max"), ], ) def test_unknown_choice_is_rejected(self, name, raw): @@ -177,7 +214,14 @@ class TestClosedChoices: @pytest.mark.parametrize( ("name", "raw"), - [("EMBEDDINGS_BATCH_SIZE", 0), ("COMPRESSION_THRESHOLD_PERCENTAGE", 1.5), ("UPLOAD_MAX_FILE_BYTES", 0)], + [ + ("EMBEDDINGS_BATCH_SIZE", 0), + ("COMPRESSION_THRESHOLD_PERCENTAGE", 1.5), + ("UPLOAD_MAX_FILE_BYTES", 0), + ("MESSAGE_EVENTS_RETENTION_DAYS", 0), + ("REMOTE_DEVICE_CMD_QUEUE_TTL_SECONDS", 605), + ("GRAPHRAG_MAX_CHUNKS_FOR_EXTRACTION", -1), + ], ) def test_out_of_range_numbers_are_rejected(self, name, raw): with pytest.raises(ValidationError):