feat: air-gapped deployment guide, no implicit downloads

- Ship tiktoken's cl100k_base inside the package and build the encoding
  from it, so token counting never downloads anything.
- Default EMBEDDINGS_CACHE_DIR to <data home>/models instead of FastEmbed's
  temp dir, and read tokenizer.json and repo metadata from that cache, so
  a model downloads once and survives reboots.
- TTS_PROVIDER=none and STT_PROVIDER=none switch the speech features off:
  the endpoints return 404, audio files fail to ingest with a clear
  message, /api/config reports tts_available/stt_available, and the UI
  hides the Speak and microphone buttons.
- Drop the Google Fonts Roboto import from the web UI.
- prefetch-models fills the cache the app reads; verify-offline checks the
  packaged encoding.
- Docs: new Air-Gapped Deployment guide, settings and cache notes.

docsgpt/scripts/prefetch_models.py

@@ -1,14 +1,13 @@
-"""Download the model artifacts a fresh container would otherwise fetch.
+"""Download the embedding models a fresh install would otherwise fetch.
 
 Run at image build time so a fresh container does not download on its first
-request, and an air-gapped install works at all. Two things are warmed:
-
-* Embedding models, into FastEmbed's cache. Both the legacy and the current
-  default are baked: an upgraded deployment keeps using mpnet until it runs
-  ``reembed``, while a new one starts on granite.
-* tiktoken's ``cl100k_base`` encoding, which token accounting uses on every
-  chat. tiktoken caches it under ``TIKTOKEN_CACHE_DIR`` (a temp dir when
-  unset), so the image sets that variable and this warms it.
+request, and before moving an install onto a host without internet access.
+Both the legacy and the current default embedding model are fetched: an
+upgraded deployment keeps using mpnet until it runs ``reembed``, while a new
+one starts on granite. Each model lands in ``EMBEDDINGS_CACHE_DIR`` together
+with its tokenizer, which chunking reads from the same snapshot. tiktoken's
+``cl100k_base`` encoding ships inside the package, so there is nothing to warm
+for it.
 
 Usage::
 
@@ -34,23 +33,6 @@ logger = logging.getLogger("prefetch_models")
 #: Fetched when no names are given.
 DEFAULT_MODELS = (DEFAULT_LEGACY, DEFAULT_NEW_INSTALL)
 
-#: tiktoken encodings the application loads (``docsgpt.utils.get_encoding``).
-TIKTOKEN_ENCODINGS = ("cl100k_base",)
-
-
-def prefetch_tiktoken(names: Sequence[str] = TIKTOKEN_ENCODINGS) -> List[str]:
-    """Warm tiktoken's cache for each encoding in ``names``.
-
-    Returns:
-        The encodings fetched.
-    """
-    import tiktoken
-
-    for name in names:
-        logger.info("Fetching tiktoken encoding %s", name)
-        tiktoken.get_encoding(name)
-    return list(names)
-
 
 def prefetch(names: Sequence[str], cache_dir: Optional[str] = None) -> List[str]:
     """Fetch each named model's artifacts.
@@ -97,6 +79,22 @@ def prefetch(names: Sequence[str], cache_dir: Optional[str] = None) -> List[str]
     return fetched
 
 
+def _cache_dir() -> Optional[str]:
+    """``EMBEDDINGS_CACHE_DIR`` from the environment, else the directory the app reads.
+
+    The image build sets the variable and copies in only this module's imports,
+    so settings are loaded only when the variable is absent.
+    """
+    import os
+
+    configured = os.environ.get("EMBEDDINGS_CACHE_DIR")
+    if configured:
+        return configured
+    from docsgpt.core.settings import settings
+
+    return settings.EMBEDDINGS_CACHE_DIR or None
+
+
 def _parse(argv: Optional[Sequence[str]], prog: str, description: str) -> list[str]:
     import argparse
 
@@ -109,13 +107,11 @@ def _parse(argv: Optional[Sequence[str]], prog: str, description: str) -> list[s
 
 def main(argv: Optional[Sequence[str]] = None) -> int:
     logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
-    import os
 
-    names = _parse(argv, "prefetch-models", "Download the embedding models and the tiktoken encodings into the local caches.")
-    fetched = prefetch(names, os.environ.get("EMBEDDINGS_CACHE_DIR"))
-    logger.info("Cached %d model(s): %s", len(fetched), ", ".join(fetched))
-    encodings = prefetch_tiktoken()
-    logger.info("Cached tiktoken encoding(s): %s", ", ".join(encodings))
+    names = _parse(argv, "prefetch-models", "Download the embedding models and their tokenizers into the model cache.")
+    cache_dir = _cache_dir()
+    fetched = prefetch(names, cache_dir)
+    logger.info("Cached %d model(s) in %s: %s", len(fetched), cache_dir or "FastEmbed's default cache", ", ".join(fetched))
     return 0