From aa0e4ea2808f7d88eff3ce3eb65b52df714a8661 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 15 Sep 2026 22:57:00 +0100 Subject: [PATCH] feat: docsgpt up runs and manages DocsGPT on Docker `docsgpt up` copies the standalone Compose file shipped with this package version into the stack directory (~/.docsgpt/server by default), writes its .env and starts the stack on the images of the same version. A first run asks who should reach DocsGPT (this computer, the network with a token, or a domain with HTTPS) and which model provider to use; flags answer the same questions for scripts. Re-running keeps secrets and settings and moves the image tag, and the database password is only generated for a new database. Also: down, status, logs, token, open, env, upgrade (uv tool installs upgrade themselves and run `up` again) and uninstall (keeps settings and data unless --purge). The commands import no Flask, Celery or settings. The wheel carries deployment/docker-compose-standalone.yaml as docsgpt/deploy/docker-compose.yaml; the sdist includes the source file. --- .github/workflows/package-build.yml | 2 + docsgpt/cli.py | 86 ++++++- docsgpt/deploy/__init__.py | 5 + docsgpt/deploy/commands.py | 381 ++++++++++++++++++++++++++++ docsgpt/deploy/docker.py | 143 +++++++++++ docsgpt/deploy/envfile.py | 74 ++++++ docsgpt/deploy/stack.py | 184 ++++++++++++++ pyproject.toml | 8 +- tests/deploy/__init__.py | 0 tests/deploy/test_commands.py | 286 +++++++++++++++++++++ tests/deploy/test_docker.py | 133 ++++++++++ tests/deploy/test_envfile.py | 72 ++++++ tests/deploy/test_stack.py | 196 ++++++++++++++ 13 files changed, 1567 insertions(+), 3 deletions(-) create mode 100644 docsgpt/deploy/__init__.py create mode 100644 docsgpt/deploy/commands.py create mode 100644 docsgpt/deploy/docker.py create mode 100644 docsgpt/deploy/envfile.py create mode 100644 docsgpt/deploy/stack.py create mode 100644 tests/deploy/__init__.py create mode 100644 tests/deploy/test_commands.py create mode 100644 tests/deploy/test_docker.py create mode 100644 tests/deploy/test_envfile.py create mode 100644 tests/deploy/test_stack.py diff --git a/.github/workflows/package-build.yml b/.github/workflows/package-build.yml index 0be880db..23798bfe 100644 --- a/.github/workflows/package-build.yml +++ b/.github/workflows/package-build.yml @@ -65,6 +65,8 @@ jobs: names = set(zipfile.ZipFile(glob.glob("dist/*.whl")[0]).namelist()) for required in ( "docsgpt/cli.py", + "docsgpt/deploy/commands.py", + "docsgpt/deploy/docker-compose.yaml", "docsgpt/alembic.ini", "docsgpt/alembic/env.py", "docsgpt/alembic/script.py.mako", diff --git a/docsgpt/cli.py b/docsgpt/cli.py index 36dc5de8..873cd099 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -1,4 +1,5 @@ -"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts. +"""The ``docsgpt`` command: run the API, the worker and the maintenance scripts, +or run and manage DocsGPT on Docker (``docsgpt up``). Every subcommand imports what it needs when it runs, so ``docsgpt --help`` stays instant and does not touch the database. @@ -153,6 +154,17 @@ def _migrate(args: argparse.Namespace) -> int: return 0 +def _deploy(name: str): + """A subcommand handler that imports ``docsgpt.deploy.commands`` only when it runs.""" + + def handler(args: argparse.Namespace, context=None) -> int: + from docsgpt.deploy import commands + + return getattr(commands, name)(args, context) + + return handler + + # Maintenance scripts keep their own argument parsers; the command hands # everything after the script name to them untouched (argparse would try to # interpret the options itself). @@ -169,11 +181,70 @@ def _run_script(module: str, argv: list[str]) -> int: return int(importlib.import_module(f"docsgpt.scripts.{module}").main(argv) or 0) +def _add_deploy_commands(commands) -> None: + """``docsgpt up`` and the commands that manage the Docker stack it runs.""" + from docsgpt.deploy.stack import EXPOSURES, PROVIDERS + + def stack_command(name: str, handler: str, help_text: str) -> argparse.ArgumentParser: + parser = commands.add_parser(name, help=help_text) + parser.add_argument("--dir", help="stack directory (default: DOCSGPT_HOME, else ~/.docsgpt/server)") + parser.set_defaults(func=_deploy(handler), deploy=True) + return parser + + up = stack_command("up", "up", "install or update DocsGPT on Docker and start it") + up.add_argument("--expose", choices=EXPOSURES, help="who can reach it: local (default), network or domain") + up.add_argument("--domain", help="public domain served over HTTPS by Caddy (implies --expose domain)") + up.add_argument("--port", type=int, help="host port for the UI and API (default: 7091)") + up.add_argument("--provider", choices=list(PROVIDERS), help="model provider (default: the DocsGPT public API)") + up.add_argument("--api-key", help="the provider's API key (or set DOCSGPT_API_KEY)") + up.add_argument("--model", help="model name (required for openai-compatible)") + up.add_argument("--base-url", help="base URL of an OpenAI-compatible server") + docling = up.add_mutually_exclusive_group() + docling.add_argument("--docling", dest="docling", action="store_const", const=True, + help="run the image with the docling parser engine and OCR (several GB larger)") + docling.add_argument("--no-docling", dest="docling", action="store_const", const=False, + help="go back to the default image") + up.set_defaults(docling=None) + up.add_argument("--image-tag", help="image tag to run instead of this package's version, e.g. develop") + up.add_argument("-y", "--yes", action="store_true", help="ask nothing: use the flags, then the defaults") + up.add_argument("--reconfigure", action="store_true", help="ask the setup questions again") + up.add_argument("--adopt", action="store_true", help="take over a DocsGPT stack started from another folder") + up.add_argument("--no-open", action="store_true", help="do not open the browser after the first install") + up.add_argument("--timeout", type=int, default=300, help="seconds to wait for the API to answer (default: 300)") + + stack_command("down", "down", "stop the Docker stack (data and settings stay)") + stack_command("status", "status", "show the stack's version, address, containers and health") + + logs = stack_command("logs", "logs", "show the stack's logs") + logs.add_argument("-f", "--follow", action="store_true", help="keep printing new lines") + logs.add_argument("--tail", type=int, help="only the last N lines of each service") + logs.add_argument("services", nargs="*", help="services to show, e.g. backend worker") + + stack_command("token", "token", "print the access token (installs reachable beyond this computer)") + stack_command("open", "open_ui", "open DocsGPT in the browser") + + upgrade = stack_command("upgrade", "upgrade", "upgrade the package and restart the stack on the new version") + upgrade.add_argument("--version", help="version to install (default: the latest release)") + + uninstall = stack_command("uninstall", "uninstall", "remove the Docker stack") + uninstall.add_argument("-y", "--yes", action="store_true", help="do not ask for confirmation") + uninstall.add_argument("--purge", action="store_true", help="also delete the settings and all data") + + env = stack_command("env", "env", "show, get or set the stack's settings") + env_actions = env.add_subparsers(dest="env_action", metavar="") + get = env_actions.add_parser("get", help="print one setting") + get.add_argument("key") + set_ = env_actions.add_parser("set", help="set settings (KEY=VALUE ...); run `docsgpt up` to apply") + set_.add_argument("pairs", nargs="+", metavar="KEY=VALUE") + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="docsgpt", description="DocsGPT: private AI for agents, assistants and search.") parser.add_argument("--version", action="version", version=f"docsgpt {__version__}") commands = parser.add_subparsers(dest="command", metavar="") + _add_deploy_commands(commands) + api = commands.add_parser("api", help="serve the HTTP API") api.add_argument("--host", default=DEFAULT_HOST, help="interface to listen on (default: localhost; 0.0.0.0 for all)") api.add_argument("--port", type=int, default=DEFAULT_PORT) @@ -212,7 +283,18 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if not args.command: parser.print_help() return 2 - return args.func(args) + if not getattr(args, "deploy", False): + return args.func(args) + from docsgpt.deploy.docker import DeployError + + try: + return args.func(args) + except DeployError as exc: + print(f"docsgpt: {exc}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print(file=sys.stderr) + return 130 if __name__ == "__main__": diff --git a/docsgpt/deploy/__init__.py b/docsgpt/deploy/__init__.py new file mode 100644 index 00000000..18c30821 --- /dev/null +++ b/docsgpt/deploy/__init__.py @@ -0,0 +1,5 @@ +"""Run DocsGPT on Docker from the installed package: ``docsgpt up`` and the commands that manage it. + +Nothing here imports the Flask app, Celery or the settings module, so these +commands start instantly and work before any configuration exists. +""" diff --git a/docsgpt/deploy/commands.py b/docsgpt/deploy/commands.py new file mode 100644 index 00000000..140b7929 --- /dev/null +++ b/docsgpt/deploy/commands.py @@ -0,0 +1,381 @@ +"""``docsgpt up`` and the commands that manage the stack it starts.""" + +from __future__ import annotations + +import getpass +import json +import os +import re +import shutil +import subprocess +import sys +import webbrowser +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from docsgpt.deploy import envfile, stack +from docsgpt.deploy.docker import DeployError, Docker, lan_ip, wait_healthy + +PROJECT = "docsgpt" +DATABASE_VOLUME = f"{PROJECT}_postgres_data" +_MOVING_TAGS = ("latest", "develop") +_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +EXPOSURE_CHOICES = [ + ("local", "Only this computer"), + ("network", "Other machines on the network (plain HTTP, access token)"), + ("domain", "A domain name with HTTPS (Caddy certificate, access token)"), +] + + +class Prompter: + """Questions on the terminal. The installer hands its terminal to ``docsgpt up``.""" + + def choose(self, question: str, options: list[tuple[str, str]], default: str) -> str: + keys = [key for key, _ in options] + print(question) + for number, (key, label) in enumerate(options, 1): + print(f" {number}) {label}{' (default)' if key == default else ''}") + while True: + answer = input(f"Choose 1-{len(options)} [{keys.index(default) + 1}]: ").strip() + if not answer: + return default + if answer.isdigit() and 1 <= int(answer) <= len(options): + return keys[int(answer) - 1] + if answer in keys: + return answer + print("Enter one of the numbers above.") + + def text(self, question: str, default: Optional[str] = None, secret: bool = False) -> str: + ask = getpass.getpass if secret else input + while True: + answer = ask(f"{question}{f' [{default}]' if default else ''}: ").strip() + if answer: + return answer + if default: + return default + + def confirm(self, question: str, default: bool = False) -> bool: + answer = input(f"{question} [{'Y/n' if default else 'y/N'}]: ").strip().lower() + return default if not answer else answer in ("y", "yes") + + +def detect_installer() -> str: + """How this ``docsgpt`` was installed: ``uv`` (uv tool), ``pipx`` or ``pip``.""" + prefix = Path(sys.prefix) + if (prefix / "uv-receipt.toml").is_file(): + return "uv" + if "pipx" in prefix.parts: + return "pipx" + return "pip" + + +def _run_command(args: list[str]) -> int: + try: + return subprocess.call(args) + except FileNotFoundError as exc: + raise DeployError(f"{args[0]} is not on PATH") from exc + + +def _exec_command(argv: list[str]) -> int: + """Replace this process with ``argv`` (the upgraded ``docsgpt``); Windows runs it and waits.""" + executable = shutil.which(argv[0]) or argv[0] + if sys.platform == "win32": + return subprocess.call([executable, *argv[1:]]) + os.execv(executable, [executable, *argv[1:]]) + return 0 # not reached + + +@dataclass +class Context: + """What the commands talk to; tests replace the parts that touch Docker, the terminal or the network.""" + + docker: Any + prompter: Any + interactive: bool + version: str + lan_ip: Callable[[], str] = lan_ip + wait: Callable[[str, float], bool] = wait_healthy + open_browser: Callable[[str], Any] = webbrowser.open + installer: Callable[[], str] = detect_installer + run: Callable[[list[str]], int] = _run_command + exec_up: Callable[[list[str]], int] = _exec_command + + @classmethod + def default(cls, args) -> "Context": + from docsgpt.version import __version__ + + interactive = sys.stdin.isatty() and not getattr(args, "yes", False) + return cls(docker=Docker(), prompter=Prompter(), interactive=interactive, version=__version__) + + +def _installed(directory: Path) -> Optional[dict[str, str]]: + """The stack's settings, or None (with a message) when there is no install in ``directory``.""" + if not (directory / stack.COMPOSE_FILE).is_file(): + print(f"No DocsGPT install in {directory}. Run `docsgpt up` first, or pass --dir.", file=sys.stderr) + return None + return envfile.read(directory / ".env") + + +def _current_provider(env: Mapping[str, str]) -> str: + name = env.get("LLM_PROVIDER", "docsgpt") + if name == "openai" and env.get("OPENAI_BASE_URL"): + return "openai-compatible" + return name if name in stack.PROVIDERS else "docsgpt" + + +def _check_other_stacks(args, context: Context, directory: Path) -> None: + others = {path for path in context.docker.project_dirs(PROJECT) if path.resolve() != directory.resolve()} + if not others or args.adopt: + return + where = ", ".join(sorted(str(path) for path in others)) + question = ( + f"Docker already runs a DocsGPT stack started from {where}; it uses the same data volumes. " + f"Manage it from {directory} instead?" + ) + if context.interactive and context.prompter.confirm(question, default=False): + return + raise DeployError( + f"Docker already runs a DocsGPT stack started from {where}. Stop it with `docker compose down` " + f"in that folder, or run again with --adopt to manage it from {directory}." + ) + + +def _choose_provider(args, context: Context, existing: Mapping[str, str], ask: bool): + name = args.provider + if name is None and ask: + name = context.prompter.choose("Which model provider?", list(stack.PROVIDERS.items()), _current_provider(existing)) + if name is None: + return None + api_key = args.api_key or os.environ.get("DOCSGPT_API_KEY") + model, base_url = args.model, args.base_url + if context.interactive: + if name == "openai-compatible": + base_url = base_url or context.prompter.text( + "Server base URL (Ollama on this machine: http://host.docker.internal:11434/v1)" + ) + model = model or context.prompter.text("Model name") + elif name != "docsgpt" and not api_key: + api_key = context.prompter.text(f"{stack.PROVIDERS[name]} API key", secret=True) + try: + return stack.provider_settings(name, api_key=api_key, model=model, base_url=base_url) + except ValueError as exc: + raise DeployError(str(exc)) from exc + + +def up(args, context: Optional[Context] = None) -> int: + """Install or update the stack in its directory and start it.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env_path = directory / ".env" + record_path = directory / stack.RECORD_FILE + existing = envfile.read(env_path) + configured = record_path.is_file() + + context.docker.preflight(context.interactive) + _check_other_stacks(args, context, directory) + + ask = context.interactive and (not configured or args.reconfigure) + expose, domain = args.expose, args.domain + if ask and expose is None and domain is None: + expose = context.prompter.choose("Who should reach DocsGPT?", EXPOSURE_CHOICES, stack.exposure(existing)) + if expose == "domain" and not domain: + if not context.interactive: + raise DeployError("--expose domain needs --domain") + domain = context.prompter.text("Domain name (its DNS must point at this machine)", existing.get("DOCSGPT_DOMAIN")) + provider = _choose_provider(args, context, existing, ask) + + image_tag = args.image_tag or context.version + try: + updates = stack.plan( + existing, + image_tag=image_tag, + fresh_database=not context.docker.volume_exists(DATABASE_VOLUME), + expose=expose, + domain=domain, + port=args.port, + provider=provider, + docling=args.docling, + ) + except ValueError as exc: + raise DeployError(str(exc)) from exc + + directory.mkdir(parents=True, exist_ok=True) + shutil.copyfile(stack.compose_source(), directory / stack.COMPOSE_FILE) + envfile.update(env_path, updates) + env = envfile.read(env_path) + + up_args = ["up", "-d", "--remove-orphans"] + if image_tag in _MOVING_TAGS: + up_args += ["--pull", "always"] + print(f"Starting DocsGPT {image_tag} from {directory} ...") + context.docker.compose(directory, *up_args) + + health = stack.health_url(env) + print("Waiting for DocsGPT to answer (the first start also sets up the database) ...") + if not context.wait(health, args.timeout): + print( + f"DocsGPT did not answer at {health} within {args.timeout} seconds. " + "See what happened with `docsgpt logs backend`.", + file=sys.stderr, + ) + return 1 + + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + record = json.loads(record_path.read_text(encoding="utf-8")) if configured else {"installed_at": now} + record.update(version=context.version, image_tag=image_tag, updated_at=now) + record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + + address = stack.url(env, context.lan_ip()) + mode = stack.exposure(env) + print(f"\nDocsGPT is running at {address}") + if env.get("AUTH_TYPE") == "simple_jwt" and env.get("JWT_SECRET_KEY"): + print("Access token (the page asks for it; `docsgpt token` prints it again):") + print(f" {stack.simple_jwt_token(env['JWT_SECRET_KEY'])}") + if mode == "network": + print("Traffic is plain HTTP. Outside a trusted network, use a domain with HTTPS: docsgpt up --domain ") + elif mode == "domain": + print("Caddy gets the certificate when it starts: DNS must point at this machine and ports 80 and 443 be open.") + print(f"Settings: {env_path} (change the model provider or access with `docsgpt up --reconfigure`)") + print("Manage it with: docsgpt status | logs | upgrade | down | uninstall") + if context.interactive and not configured and not args.no_open and mode == "local": + context.open_browser(address) + return 0 + + +def down(args, context: Optional[Context] = None) -> int: + """Stop the stack; data and settings stay.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + context.docker.compose(directory, "down") + return 0 + + +def status(args, context: Optional[Context] = None) -> int: + """Version, address, containers and whether the API answers (exit 1 when it does not).""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + tag = env.get("DOCSGPT_IMAGE_TAG", "unknown") + env.get("DOCSGPT_IMAGE_VARIANT", "") + print(f"DocsGPT {tag} in {directory}") + print(f"Address: {stack.url(env, context.lan_ip())} ({stack.exposure(env)})") + context.docker.compose(directory, "ps", check=False) + healthy = context.wait(stack.health_url(env), 0) + print("API: answering" if healthy else "API: not answering (see `docsgpt logs backend`)") + return 0 if healthy else 1 + + +def logs(args, context: Optional[Context] = None) -> int: + """``docker compose logs`` for the stack.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + options = (["--follow"] if args.follow else []) + (["--tail", str(args.tail)] if args.tail else []) + return context.docker.compose(directory, "logs", *options, *args.services, check=False).returncode + + +def token(args, context: Optional[Context] = None) -> int: + """Print the access token of a ``simple_jwt`` install.""" + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + if env.get("AUTH_TYPE") != "simple_jwt" or not env.get("JWT_SECRET_KEY"): + print(f"This install has no access token (AUTH_TYPE={env.get('AUTH_TYPE') or 'none'}).", file=sys.stderr) + return 1 + print(stack.simple_jwt_token(env["JWT_SECRET_KEY"])) + return 0 + + +def open_ui(args, context: Optional[Context] = None) -> int: + """Open DocsGPT in the browser.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + env = _installed(directory) + if env is None: + return 1 + address = stack.url(env, context.lan_ip()) + print(address) + context.open_browser(address) + return 0 + + +def env(args, context: Optional[Context] = None) -> int: + """Show where the settings are, or get and set them.""" + env_path = stack.stack_dir(args.dir) / ".env" + if args.env_action is None: + print(env_path) + return 0 + if args.env_action == "get": + values = envfile.read(env_path) + if args.key not in values: + print(f"{args.key} is not set in {env_path}", file=sys.stderr) + return 1 + print(values[args.key]) + return 0 + updates = {} + for pair in args.pairs: + key, separator, value = pair.partition("=") + if not separator or not _KEY.match(key): + raise DeployError(f"expected KEY=VALUE, got {pair!r}") + updates[key] = value + try: + envfile.update(env_path, updates) + except ValueError as exc: + raise DeployError(str(exc)) from exc + print(f"Saved to {env_path}. Run `docsgpt up` to apply.") + return 0 + + +def _uninstall_hint(installer: str) -> str: + return {"uv": "uv tool uninstall docsgpt", "pipx": "pipx uninstall docsgpt"}.get(installer, "pip uninstall docsgpt") + + +def upgrade(args, context: Optional[Context] = None) -> int: + """Upgrade the package, then run the new version's ``docsgpt up``.""" + context = context or Context.default(args) + spec = f"docsgpt=={args.version}" if args.version else "docsgpt" + installer = context.installer() + if installer == "uv": + if context.run(["uv", "tool", "install", "--force", spec]) != 0: + raise DeployError(f"uv could not install {spec}") + return context.exec_up(["docsgpt", "up", "--dir", str(stack.stack_dir(args.dir))]) + command = f"pipx install --force {spec}" if installer == "pipx" else f"pip install -U {spec}" + print(f"Upgrade the package with `{command}`, then run `docsgpt up` to move the stack to it.", file=sys.stderr) + return 1 + + +def uninstall(args, context: Optional[Context] = None) -> int: + """Remove the containers and the stack files; ``--purge`` also deletes settings and data.""" + context = context or Context.default(args) + directory = stack.stack_dir(args.dir) + if _installed(directory) is None: + return 1 + if args.purge: + what = "containers, settings and data (documents, conversations, the database)" + else: + what = "containers (the settings in .env and the data volumes are kept)" + if not args.yes: + if not context.interactive: + raise DeployError("uninstall needs --yes when there is no terminal to confirm on") + if not context.prompter.confirm(f"Remove the DocsGPT {what} in {directory}?", default=False): + print("Nothing removed.") + return 1 + context.docker.compose(directory, "down", "--remove-orphans", *(["-v"] if args.purge else [])) + if args.purge: + shutil.rmtree(directory) + print(f"Removed DocsGPT and its data from {directory}.") + else: + for name in (stack.COMPOSE_FILE, stack.RECORD_FILE): + (directory / name).unlink(missing_ok=True) + print(f"Removed the containers. Settings stay in {directory / '.env'} and data in the Docker volumes.") + print(f"To remove the docsgpt command too: {_uninstall_hint(context.installer())}") + return 0 diff --git a/docsgpt/deploy/docker.py b/docsgpt/deploy/docker.py new file mode 100644 index 00000000..0b3a1ea4 --- /dev/null +++ b/docsgpt/deploy/docker.py @@ -0,0 +1,143 @@ +"""The docker CLI calls behind ``docsgpt up``.""" + +from __future__ import annotations + +import http.client +import re +import shutil +import socket +import subprocess +import sys +import time +import urllib.request +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Optional + +MIN_COMPOSE = (2, 24, 0) +DAEMON_START_SECONDS = 120 + + +class DeployError(Exception): + """A problem the user can act on; the command prints it without a traceback.""" + + +def run(args: Sequence[str], *, cwd: Optional[Path] = None, capture: bool = False, check: bool = True): + """Run a command, streaming its output unless ``capture``; with ``check`` a failure raises DeployError.""" + try: + result = subprocess.run(list(args), cwd=cwd, text=True, capture_output=capture, check=False) + except FileNotFoundError as exc: + raise DeployError(f"{args[0]} is not installed or not on PATH") from exc + if check and result.returncode != 0: + detail = (result.stderr or "").strip() if capture else "" + message = f"`{' '.join(args)}` failed with exit code {result.returncode}" + raise DeployError(f"{message}: {detail}" if detail else message) + return result + + +def _parse_version(text: str) -> Optional[tuple[int, ...]]: + match = re.search(r"(\d+)\.(\d+)\.(\d+)", text or "") + return tuple(int(part) for part in match.groups()) if match else None + + +class Docker: + """Docker and Docker Compose, through their command-line tools.""" + + def __init__( + self, + runner: Callable[..., subprocess.CompletedProcess] = run, + which: Callable[[str], Optional[str]] = shutil.which, + sleep: Callable[[float], None] = time.sleep, + platform: str = sys.platform, + ) -> None: + self._run = runner + self._which = which + self._sleep = sleep + self._platform = platform + + def preflight(self, interactive: bool = False) -> None: + """Make sure Docker is installed and running and Compose is new enough, starting Docker Desktop on macOS.""" + if not self._which("docker"): + raise DeployError("Docker is not installed. Get it from https://docs.docker.com/get-docker/ and run this again.") + if not self.daemon_running(): + self._start_daemon() + result = self._run(["docker", "compose", "version", "--short"], capture=True, check=False) + version = _parse_version(result.stdout) if result.returncode == 0 else None + if version is None: + raise DeployError( + "Docker Compose v2 is not available (`docker compose version` failed). " + "Install the Compose plugin: https://docs.docker.com/compose/install/" + ) + if version < MIN_COMPOSE: + found = ".".join(str(part) for part in version) + raise DeployError(f"Docker Compose {found} is too old; DocsGPT needs 2.24 or newer.") + + def daemon_running(self) -> bool: + return self._run(["docker", "info"], capture=True, check=False).returncode == 0 + + def _start_daemon(self) -> None: + if self._platform == "darwin": + print("Docker is not running; starting Docker Desktop ...", file=sys.stderr) + self._run(["open", "-a", "Docker"], check=False) + for _ in range(DAEMON_START_SECONDS // 2): + self._sleep(2) + if self.daemon_running(): + return + raise DeployError("Docker Desktop did not start within two minutes. Start it and run this again.") + if self._platform.startswith("linux"): + raise DeployError("Docker is not running. Start it with `sudo systemctl start docker` and run this again.") + raise DeployError("Docker is not running. Start Docker Desktop and run this again.") + + def compose(self, directory: Path, *args: str, capture: bool = False, check: bool = True): + """``docker compose `` in ``directory``, which holds the Compose file and its ``.env``.""" + return self._run(["docker", "compose", *args], cwd=directory, capture=capture, check=check) + + def volume_exists(self, name: str) -> bool: + return self._run(["docker", "volume", "inspect", name], capture=True, check=False).returncode == 0 + + def project_dirs(self, project: str) -> set[Path]: + """The folders containers of Compose project ``project`` were started from.""" + result = self._run( + [ + "docker", "ps", "-a", + "--filter", f"label=com.docker.compose.project={project}", + "--format", '{{.Label "com.docker.compose.project.working_dir"}}', + ], + capture=True, + check=False, + ) + if result.returncode != 0: + return set() + return {Path(line.strip()) for line in result.stdout.splitlines() if line.strip()} + + +def wait_healthy( + url: str, + timeout: float, + *, + opener: Callable = urllib.request.urlopen, + sleep: Callable[[float], None] = time.sleep, + clock: Callable[[], float] = time.monotonic, +) -> bool: + """Poll ``url`` until it answers 2xx (True) or ``timeout`` seconds pass (False); tries at least once.""" + deadline = clock() + timeout + while True: + try: + with opener(url, timeout=5) as response: + if 200 <= response.status < 300: + return True + except (OSError, http.client.HTTPException): + pass + if clock() >= deadline: + return False + sleep(2) + + +def lan_ip() -> str: + """This machine's address on its network, or ``localhost``. No packet is sent.""" + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe: + try: + probe.connect(("192.0.2.1", 80)) + return probe.getsockname()[0] + except OSError: + return "localhost" diff --git a/docsgpt/deploy/envfile.py b/docsgpt/deploy/envfile.py new file mode 100644 index 00000000..9d51b946 --- /dev/null +++ b/docsgpt/deploy/envfile.py @@ -0,0 +1,74 @@ +"""Read and update a ``.env`` file in place, keeping the lines the user wrote.""" + +from __future__ import annotations + +import os +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Optional + +_ASSIGNMENT = re.compile(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=(.*)$") +# Values written without quotes: nothing Compose would interpolate, strip or treat as a comment. +_PLAIN = re.compile(r"^[A-Za-z0-9_./:@+,=-]*$") + + +def _parse_value(raw: str) -> str: + """The value of one assignment, with Compose's quoting rules.""" + value = raw.strip() + if len(value) >= 2 and value[0] == value[-1] == "'": + return value[1:-1] + if len(value) >= 2 and value[0] == value[-1] == '"': + return re.sub(r'\\(["\\])', r"\1", value[1:-1]) + return value.split(" #", 1)[0].rstrip() + + +def _format_value(value: str) -> str: + """``value`` quoted so it reads back unchanged.""" + if "\n" in value or "\r" in value: + raise ValueError("a .env value cannot contain a newline") + if _PLAIN.match(value): + return value + if "'" not in value: + return f"'{value}'" + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def read(path: Path) -> dict[str, str]: + """The assignments in ``path`` (empty when it does not exist); a repeated key keeps its last value.""" + path = Path(path) + if not path.is_file(): + return {} + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + match = _ASSIGNMENT.match(line) + if match: + values[match.group(1)] = _parse_value(match.group(2)) + return values + + +def update(path: Path, values: Mapping[str, Optional[str]]) -> None: + """Set each key in place (``None`` removes it), append new keys, and leave every other line alone. + + A new file is created readable by its owner only: it holds secrets. + """ + formatted = {key: None if value is None else _format_value(value) for key, value in values.items()} + path = Path(path) + lines = path.read_text(encoding="utf-8").splitlines() if path.is_file() else [] + written: set[str] = set() + out: list[str] = [] + for line in lines: + match = _ASSIGNMENT.match(line) + key = match.group(1) if match else None + if key not in formatted: + out.append(line) + continue + if key not in written and formatted[key] is not None: + out.append(f"{key}={formatted[key]}") + written.add(key) + out.extend(f"{key}={value}" for key, value in formatted.items() if key not in written and value is not None) + + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write("\n".join(out) + "\n" if out else "") diff --git a/docsgpt/deploy/stack.py b/docsgpt/deploy/stack.py new file mode 100644 index 00000000..bb577c56 --- /dev/null +++ b/docsgpt/deploy/stack.py @@ -0,0 +1,184 @@ +"""The stack ``docsgpt up`` runs: where it lives and what its ``.env`` holds.""" + +from __future__ import annotations + +import os +import secrets +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Optional + +from docsgpt.core import paths + +COMPOSE_FILE = "docker-compose.yaml" +RECORD_FILE = "install.json" +DEFAULT_PORT = 7091 +EXPOSURES = ("local", "network", "domain") + +PROVIDERS = { + "docsgpt": "DocsGPT public API (free, no key)", + "openai": "OpenAI", + "anthropic": "Anthropic", + "google": "Google Gemini", + "openrouter": "OpenRouter", + "groq": "Groq", + "openai-compatible": "OpenAI-compatible server (Ollama, vLLM, LM Studio, ...)", +} + +_LOCAL_BINDS = ("", "127.0.0.1", "localhost", "::1") +_ALL_INTERFACES = ("0.0.0.0", "::") + + +def compose_source() -> Path: + """The Compose file for this version: shipped in the wheel, or ``deployment/`` in a checkout.""" + packaged = paths.package_dir() / "deploy" / COMPOSE_FILE + if packaged.is_file(): + return packaged + root = paths.checkout_root() + if root is not None: + in_checkout = root / "deployment" / "docker-compose-standalone.yaml" + if in_checkout.is_file(): + return in_checkout + raise FileNotFoundError("the Compose file is missing from this docsgpt installation; reinstall the package") + + +def stack_dir(explicit: Optional[str]) -> Path: + """Where the stack lives: ``--dir``, else ``DOCSGPT_HOME``, else the default home (never a checkout).""" + if explicit: + return Path(explicit).expanduser().resolve() + configured = os.environ.get(paths.HOME_ENV) + if configured: + return Path(configured).expanduser().resolve() + return paths.default_home() + + +def _profiles(env: Mapping[str, str]) -> set[str]: + return {name.strip() for name in env.get("COMPOSE_PROFILES", "").split(",") if name.strip()} + + +def exposure(env: Mapping[str, str]) -> str: + """Who can reach the stack, read back from its ``.env``: local, network or domain.""" + if "https" in _profiles(env) and env.get("DOCSGPT_DOMAIN"): + return "domain" + if env.get("DOCSGPT_BIND", "") not in _LOCAL_BINDS: + return "network" + return "local" + + +def provider_settings( + name: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + base_url: Optional[str] = None, +) -> dict[str, Optional[str]]: + """The model settings for a provider choice; keys another provider used are cleared.""" + if name not in PROVIDERS: + raise ValueError(f"unknown provider {name!r}; choose one of: {', '.join(PROVIDERS)}") + if name == "docsgpt": + return {"LLM_PROVIDER": "docsgpt", "API_KEY": None, "LLM_NAME": None, "OPENAI_BASE_URL": None} + if name == "openai-compatible": + if not base_url: + raise ValueError("an OpenAI-compatible server needs a base URL, e.g. http://host.docker.internal:11434/v1") + if not model: + raise ValueError("an OpenAI-compatible server needs a model name") + return { + "LLM_PROVIDER": "openai", + "API_KEY": api_key or "not-needed", + "LLM_NAME": model, + "OPENAI_BASE_URL": base_url, + } + if not api_key: + raise ValueError(f"{PROVIDERS[name]} needs an API key") + # Without LLM_NAME the model catalog picks the provider's first model. + return {"LLM_PROVIDER": name, "API_KEY": api_key, "LLM_NAME": model or None, "OPENAI_BASE_URL": None} + + +def plan( + existing: Mapping[str, str], + *, + image_tag: str, + fresh_database: bool, + expose: Optional[str] = None, + domain: Optional[str] = None, + port: Optional[int] = None, + provider: Optional[Mapping[str, Optional[str]]] = None, + docling: Optional[bool] = None, + secret: Optional[Callable[[], str]] = None, +) -> dict[str, Optional[str]]: + """The ``.env`` changes for an ``up``: only keys that change, ``None`` for a key to remove. + + Settings the user did not ask to change are left alone, secrets are generated + once, and the database password is only set for a database that does not exist + yet (Postgres reads it when the volume is created). + """ + secret = secret or (lambda: secrets.token_hex(32)) + wanted: dict[str, Optional[str]] = {"DOCSGPT_IMAGE_TAG": image_tag} + + if domain and expose is None: + expose = "domain" + if expose is None and "DOCSGPT_BIND" not in existing and "COMPOSE_PROFILES" not in existing: + expose = "local" + if expose == "local": + wanted.update(DOCSGPT_BIND="127.0.0.1", COMPOSE_PROFILES=None, DOCSGPT_DOMAIN=None) + elif expose == "network": + wanted.update(DOCSGPT_BIND="0.0.0.0", COMPOSE_PROFILES=None, DOCSGPT_DOMAIN=None) + elif expose == "domain": + if not domain: + raise ValueError("exposing DocsGPT on a domain needs the domain name") + wanted.update(DOCSGPT_BIND="127.0.0.1", COMPOSE_PROFILES="https", DOCSGPT_DOMAIN=domain) + elif expose is not None: + raise ValueError(f"unknown exposure {expose!r}; choose one of: {', '.join(EXPOSURES)}") + if expose in ("network", "domain") and not existing.get("AUTH_TYPE"): + wanted["AUTH_TYPE"] = "simple_jwt" + + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY"): + if not existing.get(key): + wanted[key] = secret() + if not existing.get("POSTGRES_PASSWORD") and fresh_database: + wanted["POSTGRES_PASSWORD"] = secret() + if "VITE_API_STREAMING" not in existing: + wanted["VITE_API_STREAMING"] = "true" + if port is not None: + wanted["DOCSGPT_PORT"] = str(port) + if docling is not None: + wanted["DOCSGPT_IMAGE_VARIANT"] = "-docling" if docling else None + if provider is None and "LLM_PROVIDER" not in existing: + provider = provider_settings("docsgpt") + if provider: + wanted.update(provider) + + return { + key: value + for key, value in wanted.items() + if (value is None and key in existing) or (value is not None and existing.get(key) != value) + } + + +def _port(env: Mapping[str, str]) -> str: + return env.get("DOCSGPT_PORT") or str(DEFAULT_PORT) + + +def url(env: Mapping[str, str], lan_ip: str) -> str: + """The address to open DocsGPT at.""" + mode = exposure(env) + if mode == "domain": + return f"https://{env['DOCSGPT_DOMAIN']}" + if mode == "network": + bind = env.get("DOCSGPT_BIND", "") + host = lan_ip if bind in _ALL_INTERFACES else bind + return f"http://{host}:{_port(env)}" + return f"http://localhost:{_port(env)}" + + +def health_url(env: Mapping[str, str]) -> str: + """The API health check, reached from this machine whatever the exposure.""" + bind = env.get("DOCSGPT_BIND", "") + host = "127.0.0.1" if bind in _LOCAL_BINDS or bind in _ALL_INTERFACES else bind + return f"http://{host}:{_port(env)}/api/health" + + +def simple_jwt_token(secret_key: str) -> str: + """The token the API accepts under ``AUTH_TYPE=simple_jwt`` (it signs the same payload at start).""" + from jose import jwt + + return jwt.encode({"sub": "local"}, secret_key, algorithm="HS256") diff --git a/pyproject.toml b/pyproject.toml index 58493952..30f07df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,8 +194,14 @@ exclude = [ "docsgpt/vectors/", ] +# `docsgpt up` runs the standalone Compose file of its own version. The file +# stays in deployment/ (the release asset and the curl instructions use it +# there); the wheel carries a copy inside the package. +[tool.hatch.build.targets.wheel.force-include] +"deployment/docker-compose-standalone.yaml" = "docsgpt/deploy/docker-compose.yaml" + [tool.hatch.build.targets.sdist] -include = ["/docsgpt"] +include = ["/docsgpt", "/deployment/docker-compose-standalone.yaml"] artifacts = ["docsgpt/static/**"] exclude = [ "docsgpt/Dockerfile", diff --git a/tests/deploy/__init__.py b/tests/deploy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/deploy/test_commands.py b/tests/deploy/test_commands.py new file mode 100644 index 00000000..3aa80d5c --- /dev/null +++ b/tests/deploy/test_commands.py @@ -0,0 +1,286 @@ +"""`docsgpt up` and the commands that manage the stack, against a fake Docker.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from docsgpt import cli +from docsgpt.deploy import commands, envfile, stack +from docsgpt.deploy.docker import DeployError + + +class FakeDocker: + def __init__(self, volumes=(), project_dirs=()): + self.volumes = set(volumes) + self.dirs = {Path(d) for d in project_dirs} + self.calls = [] + self.preflights = 0 + + def preflight(self, interactive=False): + self.preflights += 1 + + def compose(self, directory, *args, capture=False, check=True): + self.calls.append((Path(directory), list(args))) + if args and args[0] == "down" and "-v" in args: + self.volumes.clear() + return subprocess.CompletedProcess(["docker", "compose", *args], 0, stdout="", stderr="") + + def volume_exists(self, name): + return name in self.volumes + + def project_dirs(self, project): + return set(self.dirs) + + +class FakePrompter: + def __init__(self, answers=()): + self.answers = list(answers) + self.questions = [] + + def _next(self, question): + self.questions.append(question) + if not self.answers: + raise AssertionError(f"unexpected question: {question}") + return self.answers.pop(0) + + def choose(self, question, options, default): + return self._next(question) + + def text(self, question, default=None, secret=False): + return self._next(question) + + def confirm(self, question, default=False): + return self._next(question) + + +def _context(docker=None, prompter=None, interactive=False, healthy=True, **overrides): + context = commands.Context( + docker=docker or FakeDocker(), + prompter=prompter or FakePrompter(), + interactive=interactive, + version="0.21.0", + lan_ip=lambda: "192.168.1.10", + wait=lambda url, timeout: healthy, + open_browser=lambda url: None, + ) + for key, value in overrides.items(): + setattr(context, key, value) + return context + + +def _run(argv, context): + args = cli.build_parser().parse_args(argv) + return args.func(args, context) + + +class TestUpFirstInstall: + def test_yes_installs_locally_with_the_public_api(self, tmp_path, capsys): + docker = FakeDocker() + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + + assert (tmp_path / "docker-compose.yaml").read_text() == stack.compose_source().read_text() + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_IMAGE_TAG"] == "0.21.0" + assert env["DOCSGPT_BIND"] == "127.0.0.1" + assert env["LLM_PROVIDER"] == "docsgpt" + assert env["INTERNAL_KEY"] and env["JWT_SECRET_KEY"] and env["POSTGRES_PASSWORD"] + assert docker.preflights == 1 + assert (tmp_path, ["up", "-d", "--remove-orphans"]) in docker.calls + record = json.loads((tmp_path / "install.json").read_text()) + assert record["version"] == "0.21.0" + assert "http://localhost:7091" in capsys.readouterr().out + + def test_interactive_asks_who_reaches_it_and_which_model(self, tmp_path, capsys): + prompter = FakePrompter(["network", "anthropic", "sk-ant"]) + assert _run(["up", "--dir", str(tmp_path)], _context(prompter=prompter, interactive=True)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_BIND"] == "0.0.0.0" + assert env["AUTH_TYPE"] == "simple_jwt" + assert env["LLM_PROVIDER"] == "anthropic" + assert env["API_KEY"] == "sk-ant" + out = capsys.readouterr().out + assert "http://192.168.1.10:7091" in out + assert stack.simple_jwt_token(env["JWT_SECRET_KEY"]) in out + + def test_flags_answer_the_questions(self, tmp_path): + argv = ["up", "--dir", str(tmp_path), "--domain", "docs.example.com", "--provider", "openai", "--api-key", "sk"] + assert _run(argv, _context(interactive=True)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["COMPOSE_PROFILES"] == "https" + assert env["DOCSGPT_DOMAIN"] == "docs.example.com" + assert env["LLM_PROVIDER"] == "openai" + + def test_a_missing_api_key_is_an_error_without_a_terminal(self, tmp_path): + with pytest.raises(DeployError, match="API key"): + _run(["up", "--yes", "--dir", str(tmp_path), "--provider", "openai"], _context()) + + def test_the_api_key_can_come_from_the_environment(self, tmp_path, monkeypatch): + monkeypatch.setenv("DOCSGPT_API_KEY", "sk-env") + assert _run(["up", "--yes", "--dir", str(tmp_path), "--provider", "openai"], _context()) == 0 + assert envfile.read(tmp_path / ".env")["API_KEY"] == "sk-env" + + def test_an_existing_database_keeps_its_password(self, tmp_path): + docker = FakeDocker(volumes={"docsgpt_postgres_data"}) + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) == 0 + assert "POSTGRES_PASSWORD" not in envfile.read(tmp_path / ".env") + + def test_image_tag_and_docling(self, tmp_path): + argv = ["up", "--yes", "--dir", str(tmp_path), "--image-tag", "develop", "--docling"] + docker = FakeDocker() + assert _run(argv, _context(docker)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_IMAGE_TAG"] == "develop" + assert env["DOCSGPT_IMAGE_VARIANT"] == "-docling" + # A moving tag is pulled every time, not only when missing. + assert (tmp_path, ["up", "-d", "--remove-orphans", "--pull", "always"]) in docker.calls + + def test_an_unhealthy_start_points_at_the_logs(self, tmp_path, capsys): + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context(healthy=False)) == 1 + assert "docsgpt logs" in capsys.readouterr().err + assert not (tmp_path / "install.json").exists() + + +class TestUpAgain: + def test_keeps_the_settings_and_moves_the_version(self, tmp_path): + assert _run(["up", "--yes", "--dir", str(tmp_path)], _context()) == 0 + before = envfile.read(tmp_path / ".env") + envfile.update(tmp_path / ".env", {"CUSTOM": "mine"}) + + context = _context(interactive=True, docker=FakeDocker(volumes={"docsgpt_postgres_data"})) + context.version = "0.22.0" + assert _run(["up", "--dir", str(tmp_path)], context) == 0 + after = envfile.read(tmp_path / ".env") + assert after["DOCSGPT_IMAGE_TAG"] == "0.22.0" + assert after["CUSTOM"] == "mine" + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY", "POSTGRES_PASSWORD"): + assert after[key] == before[key] + assert context.prompter.questions == [], "a configured install is not asked again" + + def test_another_projects_containers_need_consent(self, tmp_path): + docker = FakeDocker(project_dirs={"/srv/old-docsgpt"}) + with pytest.raises(DeployError, match="--adopt"): + _run(["up", "--yes", "--dir", str(tmp_path)], _context(docker)) + assert docker.calls == [] + assert _run(["up", "--yes", "--adopt", "--dir", str(tmp_path)], _context(docker)) == 0 + + def test_consent_can_be_given_at_the_prompt(self, tmp_path): + docker = FakeDocker(project_dirs={"/srv/old-docsgpt"}) + prompter = FakePrompter([True, "local", "docsgpt"]) + assert _run(["up", "--dir", str(tmp_path)], _context(docker, prompter, interactive=True)) == 0 + + +class TestManage: + @staticmethod + def _installed(tmp_path, *extra): + assert _run(["up", "--yes", "--dir", str(tmp_path), *extra], _context()) == 0 + + def test_down(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["down", "--dir", str(tmp_path)], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["down"])] + + def test_commands_on_a_missing_install(self, tmp_path, capsys): + assert _run(["status", "--dir", str(tmp_path)], _context()) == 1 + assert "docsgpt up" in capsys.readouterr().err + + def test_status(self, tmp_path, capsys): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["status", "--dir", str(tmp_path)], _context(docker)) == 0 + out = capsys.readouterr().out + assert "0.21.0" in out and "http://localhost:7091" in out + assert (tmp_path, ["ps"]) in docker.calls + + def test_logs_pass_through(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["logs", "--dir", str(tmp_path), "-f", "--tail", "50", "backend"], _context(docker)) == 0 + assert docker.calls == [(tmp_path, ["logs", "--follow", "--tail", "50", "backend"])] + + def test_token(self, tmp_path, capsys): + self._installed(tmp_path, "--expose", "network") + capsys.readouterr() + assert _run(["token", "--dir", str(tmp_path)], _context()) == 0 + secret = envfile.read(tmp_path / ".env")["JWT_SECRET_KEY"] + assert capsys.readouterr().out.strip() == stack.simple_jwt_token(secret) + + def test_no_token_without_simple_jwt(self, tmp_path, capsys): + self._installed(tmp_path) + assert _run(["token", "--dir", str(tmp_path)], _context()) == 1 + assert "AUTH_TYPE" in capsys.readouterr().err + + def test_env_get_and_set(self, tmp_path, capsys): + self._installed(tmp_path) + capsys.readouterr() + assert _run(["env", "--dir", str(tmp_path), "set", "LLM_NAME=gpt-5.5", "OCR_ENABLED=true"], _context()) == 0 + assert "docsgpt up" in capsys.readouterr().out + assert _run(["env", "--dir", str(tmp_path), "get", "LLM_NAME"], _context()) == 0 + assert capsys.readouterr().out.strip() == "gpt-5.5" + assert _run(["env", "--dir", str(tmp_path), "get", "NOPE"], _context()) == 1 + with pytest.raises(DeployError, match="KEY=VALUE"): + _run(["env", "--dir", str(tmp_path), "set", "oops"], _context()) + + def test_uninstall_keeps_data_and_settings(self, tmp_path, capsys): + self._installed(tmp_path) + docker = FakeDocker() + assert _run(["uninstall", "--yes", "--dir", str(tmp_path)], _context(docker, installer=lambda: "uv")) == 0 + assert docker.calls == [(tmp_path, ["down", "--remove-orphans"])] + assert (tmp_path / ".env").is_file(), "the database password lives there" + assert not (tmp_path / "docker-compose.yaml").exists() + assert not (tmp_path / "install.json").exists() + assert "uv tool uninstall docsgpt" in capsys.readouterr().out + + def test_uninstall_purge_removes_everything(self, tmp_path): + directory = tmp_path / "stack" + self._installed(directory) + docker = FakeDocker() + assert _run(["uninstall", "--yes", "--purge", "--dir", str(directory)], _context(docker)) == 0 + assert docker.calls == [(directory, ["down", "--remove-orphans", "-v"])] + assert not directory.exists() + + def test_uninstall_asks_first(self, tmp_path): + self._installed(tmp_path) + docker = FakeDocker() + prompter = FakePrompter([False]) + assert _run(["uninstall", "--dir", str(tmp_path)], _context(docker, prompter, interactive=True)) == 1 + assert docker.calls == [] + + +class TestUpgrade: + def test_a_uv_tool_install_upgrades_and_runs_up_again(self, tmp_path): + self_calls = [] + context = _context( + installer=lambda: "uv", + run=lambda args: self_calls.append(args) or 0, + exec_up=lambda argv: self_calls.append(["exec", *argv]) or 0, + ) + assert _run(["upgrade", "--dir", str(tmp_path), "--version", "0.22.0"], context) == 0 + assert self_calls[0] == ["uv", "tool", "install", "--force", "docsgpt==0.22.0"] + assert self_calls[1] == ["exec", "docsgpt", "up", "--dir", str(tmp_path)] + + def test_latest_when_no_version_is_given(self, tmp_path): + self_calls = [] + context = _context(installer=lambda: "uv", run=lambda args: self_calls.append(args) or 0, + exec_up=lambda argv: 0) + assert _run(["upgrade", "--dir", str(tmp_path)], context) == 0 + assert self_calls[0] == ["uv", "tool", "install", "--force", "docsgpt"] + + def test_a_pip_install_is_told_what_to_run(self, tmp_path, capsys): + context = _context(installer=lambda: "pip", run=lambda args: pytest.fail("must not run")) + assert _run(["upgrade", "--dir", str(tmp_path)], context) == 1 + err = capsys.readouterr().err + assert "pip install -U docsgpt" in err and "docsgpt up" in err + + +class TestImports: + def test_the_deploy_commands_do_not_boot_the_app(self): + code = ( + "import sys, docsgpt.cli, docsgpt.deploy.commands; " + "loaded = {m for m in sys.modules if m in ('docsgpt.app', 'docsgpt.core.settings', 'celery', 'flask')}; " + "assert not loaded, loaded" + ) + subprocess.run([sys.executable, "-c", code], cwd=Path(__file__).resolve().parents[2], check=True) diff --git a/tests/deploy/test_docker.py b/tests/deploy/test_docker.py new file mode 100644 index 00000000..3354438c --- /dev/null +++ b/tests/deploy/test_docker.py @@ -0,0 +1,133 @@ +"""The docker CLI calls behind `docsgpt up`, against a fake runner.""" + +import subprocess +from pathlib import Path + +import pytest + +from docsgpt.deploy import docker as docker_module +from docsgpt.deploy.docker import DeployError, Docker + + +class FakeRunner: + """Answers docker commands from a table of (argv prefix -> result) and records every call.""" + + def __init__(self, answers=None): + self.answers = list((answers or {}).items()) + self.calls = [] + + def __call__(self, args, *, cwd=None, capture=False, check=True): + self.calls.append((list(args), cwd)) + for prefix, answer in self.answers: + if list(args[: len(prefix)]) == list(prefix): + if callable(answer): + answer = answer() + code, out = answer + if check and code != 0: + raise DeployError(f"{' '.join(args)} failed") + return subprocess.CompletedProcess(args, code, stdout=out, stderr="") + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + +def _docker(runner, platform="linux", which=lambda name: "/usr/bin/docker"): + return Docker(runner=runner, which=which, sleep=lambda seconds: None, platform=platform) + + +class TestPreflight: + def test_docker_missing(self): + with pytest.raises(DeployError, match="Docker is not installed"): + _docker(FakeRunner(), which=lambda name: None).preflight() + + def test_daemon_down_on_linux_says_how_to_start_it(self): + runner = FakeRunner({("docker", "info"): (1, "")}) + with pytest.raises(DeployError, match="systemctl start docker"): + _docker(runner).preflight() + + def test_daemon_down_on_macos_starts_docker_desktop(self): + state = {"started": False} + + def info(): + return (0, "") if state["started"] else (1, "") + + def start(): + state["started"] = True + return (0, "") + + runner = FakeRunner( + { + ("docker", "info"): info, + ("open", "-a", "Docker"): start, + ("docker", "compose", "version"): (0, "v2.39.1-desktop.1\n"), + } + ) + _docker(runner, platform="darwin").preflight() + assert (["open", "-a", "Docker"], None) in runner.calls + + @pytest.mark.parametrize("version", ["v2.23.0", "2.20.2"]) + def test_compose_too_old(self, version): + runner = FakeRunner({("docker", "compose", "version"): (0, version + "\n")}) + with pytest.raises(DeployError, match="2.24"): + _docker(runner).preflight() + + def test_compose_missing(self): + runner = FakeRunner({("docker", "compose", "version"): (1, "")}) + with pytest.raises(DeployError, match="Docker Compose"): + _docker(runner).preflight() + + @pytest.mark.parametrize("version", ["v2.24.0", "2.39.1-desktop.1", "5.5.1"]) + def test_compose_new_enough(self, version): + runner = FakeRunner({("docker", "compose", "version"): (0, version + "\n")}) + _docker(runner).preflight() + + +class TestQueries: + def test_compose_runs_in_the_stack_directory(self, tmp_path): + runner = FakeRunner() + _docker(runner).compose(tmp_path, "up", "-d") + assert runner.calls == [(["docker", "compose", "up", "-d"], tmp_path)] + + def test_volume_exists(self): + runner = FakeRunner({("docker", "volume", "inspect", "docsgpt_postgres_data"): (0, "[]")}) + assert _docker(runner).volume_exists("docsgpt_postgres_data") + assert not _docker(FakeRunner({("docker", "volume"): (1, "")})).volume_exists("docsgpt_postgres_data") + + def test_project_directories(self): + runner = FakeRunner({("docker", "ps"): (0, "/srv/old\n/srv/old\n\n/home/me/.docsgpt/server\n")}) + assert _docker(runner).project_dirs("docsgpt") == {Path("/srv/old"), Path("/home/me/.docsgpt/server")} + args = runner.calls[0][0] + assert "label=com.docker.compose.project=docsgpt" in args + + +class TestWaitHealthy: + def test_succeeds_once_the_api_answers(self, monkeypatch): + attempts = {"n": 0} + + def opener(url, timeout): + attempts["n"] += 1 + if attempts["n"] < 3: + raise OSError("connection refused") + return _Response(200) + + clock = iter(range(0, 1000, 2)) + assert docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 60, opener=opener, + sleep=lambda s: None, clock=lambda: next(clock)) + assert attempts["n"] == 3 + + def test_gives_up_after_the_timeout(self): + def opener(url, timeout): + raise OSError("connection refused") + + clock = iter(range(0, 1000, 10)) + assert not docker_module.wait_healthy("http://127.0.0.1:7091/api/health", 30, opener=opener, + sleep=lambda s: None, clock=lambda: next(clock)) + + +class _Response: + def __init__(self, status): + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False diff --git a/tests/deploy/test_envfile.py b/tests/deploy/test_envfile.py new file mode 100644 index 00000000..d2997d1b --- /dev/null +++ b/tests/deploy/test_envfile.py @@ -0,0 +1,72 @@ +"""Reading and updating the stack's .env without losing what the user wrote.""" + +import os +import sys + +import pytest + +from docsgpt.deploy import envfile + + +class TestRead: + def test_comments_blanks_quotes_and_export(self, tmp_path): + path = tmp_path / ".env" + path.write_text( + "# settings\n" + "\n" + "LLM_PROVIDER=openai\n" + "export API_KEY='sk-123'\n" + 'LLM_NAME="gpt 5"\n' + "EMPTY=\n" + "not a line\n" + ) + assert envfile.read(path) == { + "LLM_PROVIDER": "openai", + "API_KEY": "sk-123", + "LLM_NAME": "gpt 5", + "EMPTY": "", + } + + def test_a_missing_file_is_empty(self, tmp_path): + assert envfile.read(tmp_path / ".env") == {} + + def test_the_last_duplicate_wins(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nA=2\n") + assert envfile.read(path) == {"A": "2"} + + +class TestUpdate: + def test_changes_values_in_place_and_keeps_everything_else(self, tmp_path): + path = tmp_path / ".env" + path.write_text("# my settings\nLLM_PROVIDER=openai\n\nCUSTOM=keep me\nDOCSGPT_IMAGE_TAG=0.19.0\n") + envfile.update(path, {"DOCSGPT_IMAGE_TAG": "0.21.0"}) + assert path.read_text() == "# my settings\nLLM_PROVIDER=openai\n\nCUSTOM=keep me\nDOCSGPT_IMAGE_TAG=0.21.0\n" + + def test_appends_new_keys_and_removes_none(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nB=2") + envfile.update(path, {"B": None, "C": "3"}) + assert path.read_text() == "A=1\nC=3\n" + + def test_duplicates_collapse_to_the_first_line(self, tmp_path): + path = tmp_path / ".env" + path.write_text("A=1\nX=y\nA=2\n") + envfile.update(path, {"A": "3"}) + assert path.read_text() == "A=3\nX=y\n" + + @pytest.mark.parametrize("value", ["plain", "with space", "hash # inside", "it's", 'say "hi"', "back\\slash", ""]) + def test_values_round_trip(self, tmp_path, value): + path = tmp_path / ".env" + envfile.update(path, {"VALUE": value}) + assert envfile.read(path)["VALUE"] == value + + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX permissions") + def test_a_new_file_is_private(self, tmp_path): + path = tmp_path / "stack" / ".env" + envfile.update(path, {"JWT_SECRET_KEY": "s"}) + assert oct(os.stat(path).st_mode & 0o777) == oct(0o600) + + def test_rejects_a_newline_in_a_value(self, tmp_path): + with pytest.raises(ValueError, match="newline"): + envfile.update(tmp_path / ".env", {"A": "1\n2"}) diff --git a/tests/deploy/test_stack.py b/tests/deploy/test_stack.py new file mode 100644 index 00000000..e32c2393 --- /dev/null +++ b/tests/deploy/test_stack.py @@ -0,0 +1,196 @@ +"""What `docsgpt up` writes to the stack's .env, and where the stack lives.""" + +from itertools import count +from pathlib import Path + +import pytest + +from docsgpt.core import paths +from docsgpt.deploy import stack + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _secrets(): + numbers = count(1) + return lambda: f"secret{next(numbers)}" + + +def _first_install(**overrides): + options = {"image_tag": "0.21.0", "fresh_database": True, "secret": _secrets()} + options.update(overrides) + return stack.plan({}, **options) + + +class TestFirstInstall: + def test_defaults_are_local_with_the_public_api(self): + updates = _first_install() + assert updates["DOCSGPT_IMAGE_TAG"] == "0.21.0" + assert updates["DOCSGPT_BIND"] == "127.0.0.1" + assert updates["LLM_PROVIDER"] == "docsgpt" + assert updates["VITE_API_STREAMING"] == "true" + assert "AUTH_TYPE" not in updates + + def test_secrets_are_generated_once_each(self): + updates = _first_install() + generated = {updates["INTERNAL_KEY"], updates["JWT_SECRET_KEY"], updates["POSTGRES_PASSWORD"]} + assert len(generated) == 3 + + def test_an_existing_database_keeps_its_password(self): + """Postgres reads the password only when its volume is created.""" + updates = _first_install(fresh_database=False) + assert "POSTGRES_PASSWORD" not in updates + + +class TestRerun: + def test_secrets_and_settings_are_left_alone(self): + existing = { + "DOCSGPT_IMAGE_TAG": "0.20.0", + "INTERNAL_KEY": "k", + "JWT_SECRET_KEY": "j", + "POSTGRES_PASSWORD": "p", + "VITE_API_STREAMING": "true", + "LLM_PROVIDER": "anthropic", + "API_KEY": "sk", + "DOCSGPT_BIND": "0.0.0.0", + "AUTH_TYPE": "simple_jwt", + } + updates = stack.plan(existing, image_tag="0.21.0", fresh_database=False, secret=_secrets()) + assert updates == {"DOCSGPT_IMAGE_TAG": "0.21.0"} + + def test_a_missing_password_is_not_invented_for_an_existing_database(self): + updates = stack.plan({"INTERNAL_KEY": "k", "JWT_SECRET_KEY": "j"}, image_tag="x", fresh_database=False) + assert "POSTGRES_PASSWORD" not in updates + + +class TestExposure: + def test_network_publishes_everywhere_and_turns_on_auth(self): + updates = _first_install(expose="network") + assert updates["DOCSGPT_BIND"] == "0.0.0.0" + assert updates["AUTH_TYPE"] == "simple_jwt" + assert updates.get("COMPOSE_PROFILES") is None + + def test_domain_adds_caddy_and_auth_and_keeps_the_port_local(self): + updates = _first_install(expose="domain", domain="docs.example.com") + assert updates["COMPOSE_PROFILES"] == "https" + assert updates["DOCSGPT_DOMAIN"] == "docs.example.com" + assert updates["DOCSGPT_BIND"] == "127.0.0.1" + assert updates["AUTH_TYPE"] == "simple_jwt" + + def test_domain_needs_a_domain(self): + with pytest.raises(ValueError, match="domain"): + _first_install(expose="domain") + + def test_an_existing_auth_mode_is_not_downgraded(self): + updates = stack.plan({"AUTH_TYPE": "oidc"}, image_tag="x", expose="network", fresh_database=False) + assert "AUTH_TYPE" not in updates + + def test_back_to_local_removes_the_proxy(self): + existing = {"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "docs.example.com", "DOCSGPT_BIND": "127.0.0.1"} + updates = stack.plan(existing, image_tag="x", expose="local", fresh_database=False) + assert updates["COMPOSE_PROFILES"] is None + assert updates["DOCSGPT_DOMAIN"] is None + + def test_port_and_docling(self): + updates = _first_install(port=8080, docling=True) + assert updates["DOCSGPT_PORT"] == "8080" + assert updates["DOCSGPT_IMAGE_VARIANT"] == "-docling" + assert stack.plan({"DOCSGPT_IMAGE_VARIANT": "-docling"}, image_tag="x", docling=False, fresh_database=False)[ + "DOCSGPT_IMAGE_VARIANT" + ] is None + + @pytest.mark.parametrize( + "env, mode", + [ + ({}, "local"), + ({"DOCSGPT_BIND": "0.0.0.0"}, "network"), + ({"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "d.example.com"}, "domain"), + ], + ) + def test_the_mode_is_read_back_from_the_env(self, env, mode): + assert stack.exposure(env) == mode + + +class TestProviders: + def test_switching_provider_drops_the_old_keys(self): + existing = {"LLM_PROVIDER": "openai", "API_KEY": "sk", "LLM_NAME": "m", "OPENAI_BASE_URL": "http://x/v1"} + provider = stack.provider_settings("anthropic", api_key="ak") + updates = stack.plan(existing, image_tag="x", provider=provider, fresh_database=False) + assert updates["LLM_PROVIDER"] == "anthropic" + assert updates["API_KEY"] == "ak" + assert updates["LLM_NAME"] is None + assert updates["OPENAI_BASE_URL"] is None + + def test_the_public_api_needs_no_key(self): + assert stack.provider_settings("docsgpt") == { + "LLM_PROVIDER": "docsgpt", + "API_KEY": None, + "LLM_NAME": None, + "OPENAI_BASE_URL": None, + } + + def test_a_hosted_provider_needs_a_key(self): + with pytest.raises(ValueError, match="API key"): + stack.provider_settings("openai") + + def test_an_openai_compatible_server_needs_a_url_and_a_model(self): + with pytest.raises(ValueError, match="base URL"): + stack.provider_settings("openai-compatible", model="llama3") + settings = stack.provider_settings("openai-compatible", base_url="http://host.docker.internal:11434/v1", model="llama3") + assert settings == { + "LLM_PROVIDER": "openai", + "API_KEY": "not-needed", + "LLM_NAME": "llama3", + "OPENAI_BASE_URL": "http://host.docker.internal:11434/v1", + } + + def test_an_unknown_provider(self): + with pytest.raises(ValueError, match="unknown provider"): + stack.provider_settings("nope") + + +class TestUrls: + def test_local(self): + assert stack.url({"DOCSGPT_PORT": "8080"}, lan_ip="10.0.0.5") == "http://localhost:8080" + + def test_network_uses_the_machine_address(self): + assert stack.url({"DOCSGPT_BIND": "0.0.0.0"}, lan_ip="10.0.0.5") == "http://10.0.0.5:7091" + + def test_domain(self): + env = {"COMPOSE_PROFILES": "https", "DOCSGPT_DOMAIN": "docs.example.com"} + assert stack.url(env, lan_ip="10.0.0.5") == "https://docs.example.com" + + def test_health_is_always_checked_on_this_machine(self): + assert stack.health_url({"DOCSGPT_BIND": "0.0.0.0", "DOCSGPT_PORT": "9000"}) == "http://127.0.0.1:9000/api/health" + + +class TestToken: + def test_matches_what_the_api_prints(self): + """docsgpt/app.py signs {"sub": "local"} with JWT_SECRET_KEY for AUTH_TYPE=simple_jwt.""" + from jose import jwt + + token = stack.simple_jwt_token("s3cret") + assert token == jwt.encode({"sub": "local"}, "s3cret", algorithm="HS256") + assert jwt.decode(token, "s3cret", algorithms=["HS256"]) == {"sub": "local"} + + +class TestLocations: + def test_the_stack_dir_is_never_the_checkout(self, monkeypatch, tmp_path): + monkeypatch.delenv(paths.HOME_ENV, raising=False) + monkeypatch.setattr(paths, "default_home", lambda: tmp_path / "home") + assert stack.stack_dir(None) == tmp_path / "home" + + def test_docsgpt_home_and_then_an_explicit_dir_win(self, monkeypatch, tmp_path): + monkeypatch.setenv(paths.HOME_ENV, str(tmp_path / "env-home")) + assert stack.stack_dir(None) == (tmp_path / "env-home").resolve() + assert stack.stack_dir(str(tmp_path / "flag")) == (tmp_path / "flag").resolve() + + def test_a_checkout_uses_the_deployment_compose_file(self): + assert stack.compose_source() == REPO_ROOT / "deployment" / "docker-compose-standalone.yaml" + + def test_the_packaged_compose_file_wins(self, monkeypatch, tmp_path): + packaged = tmp_path / "docsgpt" / "deploy" / "docker-compose.yaml" + packaged.parent.mkdir(parents=True) + packaged.write_text("name: docsgpt\n") + monkeypatch.setattr(paths, "package_dir", lambda: tmp_path / "docsgpt") + assert stack.compose_source() == packaged