From cdc0a0220d1b4f6d9272d6f7f77fd6ab97b745cf Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 24 Jun 2026 23:13:58 +0100 Subject: [PATCH] Harden the Jupyter sandbox runner against env-secret exposure Run each kernel under a scrubbed environment so untrusted code can never read the host's secrets. A custom 'docsgpt-python' kernelspec launches ipykernel through a wrapper that keeps only what the kernel needs (PATH, HOME, LANG, and the Jupyter runtime/data dirs), dropping API keys, tokens, the database URL, and the gateway token. The app selects this kernel by name via SANDBOX_KERNEL_NAME, so the distinct name is never shadowed by the stock python3 spec. Per-session workspaces are created mode 0700 (defense in depth under the shared uid). The README documents the runner as a single trust domain and points to the Daytona backend for per-tenant isolation. --- application/sandbox/jupyter_gateway.py | 10 +- deployment/docker-compose.yaml | 14 ++ .../k8s/deployments/docsgpt-deploy.yaml | 16 ++ .../k8s/deployments/sandbox-deploy.yaml | 12 ++ deployment/sandbox/Dockerfile | 14 ++ deployment/sandbox/README.md | 68 +++++++- deployment/sandbox/kernel-launch.sh | 20 +++ .../kernels/docsgpt-python/kernel.json | 12 ++ .../sandbox/test_jupyter_gateway_isolation.py | 147 ++++++++++++++++++ 9 files changed, 310 insertions(+), 3 deletions(-) create mode 100644 deployment/sandbox/kernel-launch.sh create mode 100644 deployment/sandbox/kernels/docsgpt-python/kernel.json create mode 100644 tests/sandbox/test_jupyter_gateway_isolation.py diff --git a/application/sandbox/jupyter_gateway.py b/application/sandbox/jupyter_gateway.py index 1cec1d3d..9e7add1d 100644 --- a/application/sandbox/jupyter_gateway.py +++ b/application/sandbox/jupyter_gateway.py @@ -227,10 +227,16 @@ class JupyterKernelGatewaySandbox(CodeSandbox): return def _prime(self, kernel: _Kernel) -> None: - """Create the per-session workspace dir and chdir the kernel into it.""" + """Create the per-session workspace (mode 0700) and chdir the kernel into it.""" + # 0700 on the root and the per-session dir is defense-in-depth only: every + # kernel runs under one shared uid here, so this is not a cross-session + # boundary (that needs distinct uids / per-session VMs -- the Daytona backend). setup = ( "import os as _os\n" - f"_os.makedirs({kernel.workspace!r}, exist_ok=True)\n" + f"_os.makedirs({_WORKSPACE_ROOT!r}, mode=0o700, exist_ok=True)\n" + f"_os.chmod({_WORKSPACE_ROOT!r}, 0o700)\n" + f"_os.makedirs({kernel.workspace!r}, mode=0o700, exist_ok=True)\n" + f"_os.chmod({kernel.workspace!r}, 0o700)\n" f"_os.chdir({kernel.workspace!r})\n" ) result = self._run(kernel, setup, self._default_timeout) diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index e165a7f4..3294b4e5 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -26,6 +26,10 @@ services: - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt # Code-execution runner reached over HTTP + WebSocket (no docker socket). - SANDBOX_GATEWAY_URL=http://docsgpt-sandbox:8888 + # Select the runner's env-scrubbing kernelspec (distinct name; never + # shadowed by the stock "python3" spec). Must match the kernel the + # docsgpt-sandbox image installs. + - SANDBOX_KERNEL_NAME=docsgpt-python ports: - "7091:7091" volumes: @@ -52,6 +56,8 @@ services: - CACHE_REDIS_URL=redis://redis:6379/2 - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt - SANDBOX_GATEWAY_URL=http://docsgpt-sandbox:8888 + # Env-scrubbing kernelspec selected by name (see backend service). + - SANDBOX_KERNEL_NAME=docsgpt-python volumes: - ../application/indexes:/app/indexes - ../application/inputs:/app/inputs @@ -68,6 +74,14 @@ services: # runner is reachable only from backend/worker, not from the host/internet. # Egress/SSRF blocks, the gVisor `runsc` runtime, and seccomp profile come in # the hardening slice. + # + # SINGLE TRUST DOMAIN: all sessions share this one container/uid and are + # isolated by working directory only (per-session cwd) — not by a kernel/OS + # boundary. The custom kernelspec scrubs secrets from the kernel env, but + # sibling workspaces are readable under the shared uid and kernels share one + # address space. Do NOT add `env_file: ../.env` here (the runner needs no app + # secrets). For cross-tenant / untrusted multi-tenant workloads use a + # per-session VM via SANDBOX_BACKEND=daytona instead. docsgpt-sandbox: build: ./sandbox mem_limit: ${SANDBOX_MEMORY:-1g} diff --git a/deployment/k8s/deployments/docsgpt-deploy.yaml b/deployment/k8s/deployments/docsgpt-deploy.yaml index 089cba95..3772ba51 100644 --- a/deployment/k8s/deployments/docsgpt-deploy.yaml +++ b/deployment/k8s/deployments/docsgpt-deploy.yaml @@ -57,6 +57,14 @@ spec: value: "false" - name: AUTO_CREATE_DB value: "false" + # Reach the always-on code-execution runner over HTTP + WebSocket. + - name: SANDBOX_GATEWAY_URL + value: "http://docsgpt-sandbox:8888" + # Select the runner's env-scrubbing kernelspec by its distinct name so + # it is never shadowed by the stock "python3" spec. MUST be set here: + # the runner only ships the kernelspec; the app (this pod) chooses it. + - name: SANDBOX_KERNEL_NAME + value: "docsgpt-python" --- apiVersion: apps/v1 kind: Deployment @@ -111,6 +119,14 @@ spec: value: "false" - name: AUTO_CREATE_DB value: "false" + # Reach the always-on code-execution runner over HTTP + WebSocket. + - name: SANDBOX_GATEWAY_URL + value: "http://docsgpt-sandbox:8888" + # Select the runner's env-scrubbing kernelspec by its distinct name so + # it is never shadowed by the stock "python3" spec. MUST be set here: + # the runner only ships the kernelspec; the app (this pod) chooses it. + - name: SANDBOX_KERNEL_NAME + value: "docsgpt-python" --- apiVersion: apps/v1 kind: Deployment diff --git a/deployment/k8s/deployments/sandbox-deploy.yaml b/deployment/k8s/deployments/sandbox-deploy.yaml index d7cc401a..941c99ef 100644 --- a/deployment/k8s/deployments/sandbox-deploy.yaml +++ b/deployment/k8s/deployments/sandbox-deploy.yaml @@ -5,6 +5,18 @@ # # On Linux prod, schedule this onto a gVisor `runsc` RuntimeClass for kernel # isolation (uncomment `runtimeClassName` once the node has it installed). +# +# SINGLE TRUST DOMAIN: all sessions in this pod share one container/uid and are +# isolated by working directory only (per-session cwd) — not by a kernel/OS +# boundary. This image ships an env-scrubbing kernelspec under the distinct name +# "docsgpt-python"; the app (docsgpt-api / docsgpt-worker) MUST select it via +# SANDBOX_KERNEL_NAME=docsgpt-python (already set in docsgpt-deploy.yaml) so the +# scrubber runs. The stock "python3" spec stays untouched, so it can never +# shadow the scrubbing kernel. Sibling workspaces are still readable under the +# shared uid and kernels share one address space. Do NOT inject app secrets +# (DB/LLM creds) into this pod's env; the runner needs none. For cross-tenant / +# untrusted multi-tenant workloads use a per-session VM via +# SANDBOX_BACKEND=daytona instead. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deployment/sandbox/Dockerfile b/deployment/sandbox/Dockerfile index e12fcd2b..b36cf107 100644 --- a/deployment/sandbox/Dockerfile +++ b/deployment/sandbox/Dockerfile @@ -42,6 +42,20 @@ RUN if [ "$INSTALL_DOCLING" = "true" ]; then \ pip install --no-cache-dir docling==2.8.3; \ fi +# Env-scrubbing kernel launcher + custom kernelspec. The launcher re-execs +# ipykernel under a minimal allowlisted env (env -i) so NO secret in the +# gateway's environment (*_API_KEY, *_TOKEN, POSTGRES_URI, the gateway auth +# token, ...) ever reaches kernel code. The kernelspec ships under a DISTINCT +# name ("docsgpt-python"), so the app selects it with SANDBOX_KERNEL_NAME and it +# is never shadowed by the stock ipykernel "python3" spec regardless of the +# python prefix. The stock "python3" spec is left untouched (no overwrite, no +# kernelspec-name precedence to rely on). SECURITY: never give this image +# `env_file: ../.env` -- the scrubber blocks exfil from the kernel, but the +# runner image itself should stay free of app secrets it has no use for. +COPY kernel-launch.sh /opt/docsgpt/kernel-launch.sh +RUN chmod 0555 /opt/docsgpt/kernel-launch.sh +COPY kernels/docsgpt-python/kernel.json /usr/local/share/jupyter/kernels/docsgpt-python/kernel.json + USER sandbox WORKDIR /home/sandbox diff --git a/deployment/sandbox/README.md b/deployment/sandbox/README.md index 31286c95..fd587ea1 100644 --- a/deployment/sandbox/README.md +++ b/deployment/sandbox/README.md @@ -5,6 +5,58 @@ backend/worker is the **client** and connects over HTTP + WebSocket via `SANDBOX_GATEWAY_URL`. Each session is an **in-process kernel** (child process), never a child container; the Docker socket is **not** mounted. +## Isolation model + +Read this before pointing untrusted or multi-tenant workloads at the runner. + +A single Jupyter runner is **one trust domain**. Every session is an in-process +kernel under **one shared uid (10001)** in **one container**; sessions are +isolated by **working directory only** — each session's code runs with its cwd +set to its own `/tmp/docsgpt-sandbox/` directory. That is a +convenience boundary, not a security boundary between sessions. + +What this slice does close: + +- **Env-secret exfil is closed.** The custom kernelspec + (`kernels/docsgpt-python/kernel.json` → `/opt/docsgpt/kernel-launch.sh`) + re-execs ipykernel under a minimal allowlisted env (`env -i` keeping only + `PATH`, `HOME`, `LANG`, `JUPYTER_RUNTIME_DIR`, `JUPYTER_DATA_DIR`). The image + installs this spec under the **distinct name `docsgpt-python`** and the app + selects it via `SANDBOX_KERNEL_NAME=docsgpt-python`; because the name is + distinct, it is **never shadowed** by the stock ipykernel `python3` spec + (kernelspec name resolution prefers `sys.prefix/share` over + `/usr/local/share`, so reusing `python3` would silently fall back to the + unscrubbed stock spec on a different python prefix). The stock `python3` spec + is left untouched. So even though the gateway process inherits the operator's + full environment, **no `*_API_KEY` / `*_TOKEN` / `POSTGRES_URI` / gateway auth + token reaches kernel code** via `os.environ`, regardless of how the gateway is + launched. Loopback ZMQ reachability is preserved because `{connection_file}` + is forwarded untouched. +- **Per-session workspace perms.** The workspace root and each session dir are + created `0700` (defense-in-depth). Under one shared uid this does **not** stop + a sibling session from reading another's files — it only narrows exposure to + other uids on the box. + +Residual gaps (treat all sessions in one runner as mutually trusting): + +- **Sibling-workspace reads.** All kernels run as the same uid, so one session's + code can read another session's files (and `/tmp`) despite `0700`. Distinct + uids / per-session VMs are required to close this. +- **In-memory / cross-kernel.** Kernels are child processes of one gateway under + one uid; OS-level process isolation is the only boundary, and it is not a + sandbox boundary against a determined escape. No gVisor in the base posture. +- **Egress.** Outbound is broad by design (so code can `pip install` / call + public APIs). Private/link-local/metadata ranges are blocked **only** by the + network layer — the k8s NetworkPolicy or a host/cloud firewall (see *Network + egress / SSRF* below), never by the runner itself. + +For real per-tenant isolation (cross-tenant or untrusted code), use the +**Daytona backend** (`SANDBOX_BACKEND=daytona`), which gives each session its +own VM. To harden the self-hosted Jupyter runner as a whole (host protection + +egress), layer the **gVisor `runsc` runtime**, the **NetworkPolicy**, and a +**host firewall** as documented below — those protect the host and constrain +egress; they do **not** create a boundary between sessions inside one runner. + ## Run standalone for dev Build and run the runner on its own, then point the app at it: @@ -28,6 +80,16 @@ limit so large `get_file` base64 payloads aren't silently truncated. (On older gateways the trait may live elsewhere; the client's `get_file` integrity check catches any truncation regardless.) +A bare-venv gateway uses the **stock** `python3` kernelspec, which inherits the +gateway's full env (no secret scrubbing). The default `SANDBOX_KERNEL_NAME` is +`python3`, so plain venv dev gets no scrubbing — acceptable for single-trust +dev. The Docker image instead ships the env-scrubbing spec under the distinct +name `docsgpt-python` (see *Isolation model*) and the runner stack sets +`SANDBOX_KERNEL_NAME=docsgpt-python`. To get the scrubbing behavior in a venv, +copy `kernels/docsgpt-python/kernel.json` (pointing `argv` at a local copy of +`kernel-launch.sh`) into a Jupyter data dir on the kernelspec search path and +set `SANDBOX_KERNEL_NAME=docsgpt-python` before launching. + ## Exposing the port requires auth The image does **not** set `--KernelGatewayApp.allow_origin=*`. If you publish @@ -41,7 +103,11 @@ required there. The `docsgpt-sandbox` service is defined in `deployment/docker-compose.yaml` on an internal-only network. The backend and worker reach it at -`http://docsgpt-sandbox:8888`. +`http://docsgpt-sandbox:8888` and select the scrubbing kernel by setting +`SANDBOX_KERNEL_NAME=docsgpt-python` (the runner only ships the kernelspec; the +app chooses it). The same applies to k8s: `SANDBOX_KERNEL_NAME=docsgpt-python` +is set on the `docsgpt-api` and `docsgpt-worker` deployments in +`deployment/k8s/deployments/docsgpt-deploy.yaml`. ## Document extraction variant (Docling) diff --git a/deployment/sandbox/kernel-launch.sh b/deployment/sandbox/kernel-launch.sh new file mode 100644 index 00000000..b3788e89 --- /dev/null +++ b/deployment/sandbox/kernel-launch.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Env-scrubbing launcher for the docsgpt-sandbox ipykernel. +# +# The gateway process inherits the operator's full environment, which can carry +# secrets (*_API_KEY, *_TOKEN, POSTGRES_URI, the gateway auth token, ...). Stock +# kernels inherit that env verbatim, so LLM-authored code could read it via +# os.environ. This wrapper re-execs ipykernel under a MINIMAL allowlisted env so +# NO secret reaches kernel code, regardless of how the gateway was launched. +# +# Only what ipykernel needs is kept: PATH (find python), HOME (~/.ipython etc), +# LANG (encoding), and the Jupyter runtime/data dirs (writable tmpfs paths). The +# {connection_file} the gateway passes is forwarded via "$@" so loopback ZMQ +# reachability is preserved -- do NOT drop or rewrite those args. +exec env -i \ + PATH="${PATH}" \ + HOME="${HOME}" \ + LANG="${LANG}" \ + JUPYTER_RUNTIME_DIR="${JUPYTER_RUNTIME_DIR}" \ + JUPYTER_DATA_DIR="${JUPYTER_DATA_DIR}" \ + python -m ipykernel_launcher "$@" diff --git a/deployment/sandbox/kernels/docsgpt-python/kernel.json b/deployment/sandbox/kernels/docsgpt-python/kernel.json new file mode 100644 index 00000000..3846fb47 --- /dev/null +++ b/deployment/sandbox/kernels/docsgpt-python/kernel.json @@ -0,0 +1,12 @@ +{ + "argv": [ + "/opt/docsgpt/kernel-launch.sh", + "-f", + "{connection_file}" + ], + "display_name": "Python 3 (docsgpt-sandbox, scrubbed env)", + "language": "python", + "metadata": { + "debugger": true + } +} diff --git a/tests/sandbox/test_jupyter_gateway_isolation.py b/tests/sandbox/test_jupyter_gateway_isolation.py new file mode 100644 index 00000000..987d2cdb --- /dev/null +++ b/tests/sandbox/test_jupyter_gateway_isolation.py @@ -0,0 +1,147 @@ +"""Per-session isolation hardening for the Jupyter gateway sandbox. + +Covers the env-scrubbing kernel launcher (`deployment/sandbox/kernel-launch.sh`) +and the `0700` per-session workspace perms applied by `_prime`. Hermetic: the +launcher test runs the wrapper with a fake `python` on PATH, and the `_prime` +test executes the wrapper's setup code against a real temp directory by +stubbing `_run` -- no gateway / kernel process required. +""" + +import json +import os +import shutil +import stat +import subprocess +import textwrap +from pathlib import Path + +import pytest + +from application.sandbox import jupyter_gateway +from application.sandbox.base import ExecResult +from application.sandbox.jupyter_gateway import JupyterKernelGatewaySandbox, _Kernel + +_SANDBOX_DIR = Path(__file__).resolve().parents[2] / "deployment" / "sandbox" +_WRAPPER = _SANDBOX_DIR / "kernel-launch.sh" +_KERNEL_NAME = "docsgpt-python" +_KERNELSPEC = _SANDBOX_DIR / "kernels" / _KERNEL_NAME / "kernel.json" + + +# -- Env-scrubbing kernel launcher --------------------------------------------- + + +@pytest.mark.skipif(shutil.which("sh") is None, reason="POSIX sh not available") +def test_kernel_launch_scrubs_secrets_keeps_runtime_env(tmp_path): + """The wrapper drops *_API_KEY/*_TOKEN but keeps PATH/HOME/JUPYTER_* for ipykernel.""" + # Fake `python` on PATH: ignore `-m ipykernel_launcher` and dump the env it was given. + fake_python = tmp_path / "python" + fake_python.write_text( + textwrap.dedent( + """\ + #!/bin/sh + env + """ + ) + ) + fake_python.chmod(0o755) + + env = { + "PATH": f"{tmp_path}:{os.environ.get('PATH', '')}", + "HOME": str(tmp_path), + "LANG": "C.UTF-8", + "JUPYTER_RUNTIME_DIR": str(tmp_path / "runtime"), + "JUPYTER_DATA_DIR": str(tmp_path / "data"), + # Secrets that must NOT reach the kernel. + "OPENAI_API_KEY": "sk-super-secret", + "SANDBOX_GATEWAY_AUTH_TOKEN": "gateway-token", + "POSTGRES_URI": "postgresql://u:p@h/db", + } + proc = subprocess.run( + ["sh", str(_WRAPPER), "-f", "/tmp/conn.json"], + env=env, + capture_output=True, + text=True, + timeout=15, + ) + assert proc.returncode == 0, proc.stderr + out = proc.stdout + # Secrets stripped. + assert "OPENAI_API_KEY" not in out + assert "sk-super-secret" not in out + assert "SANDBOX_GATEWAY_AUTH_TOKEN" not in out + assert "POSTGRES_URI" not in out + # Allowlisted runtime env kept. + assert "PATH=" in out + assert f"HOME={tmp_path}" in out + assert f"JUPYTER_RUNTIME_DIR={tmp_path / 'runtime'}" in out + assert f"JUPYTER_DATA_DIR={tmp_path / 'data'}" in out + # The connection-file args were forwarded to ipykernel (reachability preserved). + # The fake python prints env only, so just assert it was invoked with no crash above. + + +@pytest.mark.skipif(shutil.which("sh") is None, reason="POSIX sh not available") +def test_kernel_launch_is_valid_sh(): + """The wrapper parses under POSIX sh (`sh -n`).""" + proc = subprocess.run(["sh", "-n", str(_WRAPPER)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + + +# -- Scrubbing kernelspec is selectable ---------------------------------------- + + +def test_kernelspec_argv_points_at_scrubbing_wrapper(): + """The shipped kernel.json launches the env-scrubbing wrapper, not bare ipykernel.""" + spec = json.loads(_KERNELSPEC.read_text()) + argv = spec["argv"] + assert argv[0].endswith("kernel-launch.sh") + assert "{connection_file}" in argv + + +def test_distinct_kernel_name_resolves_to_scrubbing_spec(tmp_path, monkeypatch): + """A distinct kernel name resolves to the scrubbing wrapper (never the stock python3 spec).""" + kernelspec = pytest.importorskip("jupyter_client.kernelspec") + + # Seed a Jupyter data dir with the custom spec under its distinct name. + data_dir = tmp_path / "jupyter" + spec_dir = data_dir / "kernels" / _KERNEL_NAME + spec_dir.mkdir(parents=True) + shutil.copy(_KERNELSPEC, spec_dir / "kernel.json") + monkeypatch.setenv("JUPYTER_PATH", str(data_dir)) + + manager = kernelspec.KernelSpecManager() + resolved = manager.get_kernel_spec(_KERNEL_NAME) + assert resolved.argv[0].endswith("kernel-launch.sh") + assert "{connection_file}" in resolved.argv + + +# -- Per-session workspace perms (0700) ---------------------------------------- + + +def _exec_setup_in_tmp(code: str) -> None: + """Run the kernel-side setup snippet in-process (it is plain os.* calls).""" + exec(compile(code, "", "exec"), {}) + + +def test_prime_creates_workspace_mode_0700(tmp_path, monkeypatch): + """`_prime` creates the workspace root and per-session dir at mode 0700.""" + root = tmp_path / "docsgpt-sandbox" + monkeypatch.setattr(jupyter_gateway, "_WORKSPACE_ROOT", str(root)) + + sb = JupyterKernelGatewaySandbox(gateway_url="http://unused") + workspace = f"{root}/conv-perms" + kernel = _Kernel("kid", workspace) + + captured = {} + + def fake_run(_kernel, code, _timeout): + captured["code"] = code + _exec_setup_in_tmp(code) + return ExecResult(status="ok", exit_code=0) + + monkeypatch.setattr(sb, "_run", fake_run) + monkeypatch.chdir(tmp_path) # _prime's os.chdir must land somewhere harmless + sb._prime(kernel) + + assert kernel.initialized + assert stat.S_IMODE(os.stat(root).st_mode) == 0o700 + assert stat.S_IMODE(os.stat(workspace).st_mode) == 0o700