Harden the Jupyter sandbox runner against env-secret exposure

Run each kernel under a scrubbed environment so untrusted code can never read
the host's secrets. A custom 'docsgpt-python' kernelspec launches ipykernel
through a wrapper that keeps only what the kernel needs (PATH, HOME, LANG, and
the Jupyter runtime/data dirs), dropping API keys, tokens, the database URL, and
the gateway token. The app selects this kernel by name via SANDBOX_KERNEL_NAME,
so the distinct name is never shadowed by the stock python3 spec. Per-session
workspaces are created mode 0700 (defense in depth under the shared uid). The
README documents the runner as a single trust domain and points to the Daytona
backend for per-tenant isolation.

application/sandbox/jupyter_gateway.py

@@ -227,10 +227,16 @@ class JupyterKernelGatewaySandbox(CodeSandbox):
                 return
 
     def _prime(self, kernel: _Kernel) -> None:
-        """Create the per-session workspace dir and chdir the kernel into it."""
+        """Create the per-session workspace (mode 0700) and chdir the kernel into it."""
+        # 0700 on the root and the per-session dir is defense-in-depth only: every
+        # kernel runs under one shared uid here, so this is not a cross-session
+        # boundary (that needs distinct uids / per-session VMs -- the Daytona backend).
         setup = (
             "import os as _os\n"
-            f"_os.makedirs({kernel.workspace!r}, exist_ok=True)\n"
+            f"_os.makedirs({_WORKSPACE_ROOT!r}, mode=0o700, exist_ok=True)\n"
+            f"_os.chmod({_WORKSPACE_ROOT!r}, 0o700)\n"
+            f"_os.makedirs({kernel.workspace!r}, mode=0o700, exist_ok=True)\n"
+            f"_os.chmod({kernel.workspace!r}, 0o700)\n"
             f"_os.chdir({kernel.workspace!r})\n"
         )
         result = self._run(kernel, setup, self._default_timeout)

deployment/docker-compose.yaml

@@ -26,6 +26,10 @@ services:
       - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
       # Code-execution runner reached over HTTP + WebSocket (no docker socket).
       - SANDBOX_GATEWAY_URL=http://docsgpt-sandbox:8888
+      # Select the runner's env-scrubbing kernelspec (distinct name; never
+      # shadowed by the stock "python3" spec). Must match the kernel the
+      # docsgpt-sandbox image installs.
+      - SANDBOX_KERNEL_NAME=docsgpt-python
     ports:
       - "7091:7091"
     volumes:
@@ -52,6 +56,8 @@ services:
       - CACHE_REDIS_URL=redis://redis:6379/2
       - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
       - SANDBOX_GATEWAY_URL=http://docsgpt-sandbox:8888
+      # Env-scrubbing kernelspec selected by name (see backend service).
+      - SANDBOX_KERNEL_NAME=docsgpt-python
     volumes:
       - ../application/indexes:/app/indexes
       - ../application/inputs:/app/inputs
@@ -68,6 +74,14 @@ services:
   # runner is reachable only from backend/worker, not from the host/internet.
   # Egress/SSRF blocks, the gVisor `runsc` runtime, and seccomp profile come in
   # the hardening slice.
+  #
+  # SINGLE TRUST DOMAIN: all sessions share this one container/uid and are
+  # isolated by working directory only (per-session cwd) — not by a kernel/OS
+  # boundary. The custom kernelspec scrubs secrets from the kernel env, but
+  # sibling workspaces are readable under the shared uid and kernels share one
+  # address space. Do NOT add `env_file: ../.env` here (the runner needs no app
+  # secrets). For cross-tenant / untrusted multi-tenant workloads use a
+  # per-session VM via SANDBOX_BACKEND=daytona instead.
   docsgpt-sandbox:
     build: ./sandbox
     mem_limit: ${SANDBOX_MEMORY:-1g}

deployment/k8s/deployments/docsgpt-deploy.yaml

@@ -57,6 +57,14 @@ spec:
           value: "false"
         - name: AUTO_CREATE_DB
           value: "false"
+        # Reach the always-on code-execution runner over HTTP + WebSocket.
+        - name: SANDBOX_GATEWAY_URL
+          value: "http://docsgpt-sandbox:8888"
+        # Select the runner's env-scrubbing kernelspec by its distinct name so
+        # it is never shadowed by the stock "python3" spec. MUST be set here:
+        # the runner only ships the kernelspec; the app (this pod) chooses it.
+        - name: SANDBOX_KERNEL_NAME
+          value: "docsgpt-python"
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -111,6 +119,14 @@ spec:
           value: "false"
         - name: AUTO_CREATE_DB
           value: "false"
+        # Reach the always-on code-execution runner over HTTP + WebSocket.
+        - name: SANDBOX_GATEWAY_URL
+          value: "http://docsgpt-sandbox:8888"
+        # Select the runner's env-scrubbing kernelspec by its distinct name so
+        # it is never shadowed by the stock "python3" spec. MUST be set here:
+        # the runner only ships the kernelspec; the app (this pod) chooses it.
+        - name: SANDBOX_KERNEL_NAME
+          value: "docsgpt-python"
 ---
 apiVersion: apps/v1
 kind: Deployment

deployment/k8s/deployments/sandbox-deploy.yaml

@@ -5,6 +5,18 @@
 #
 # On Linux prod, schedule this onto a gVisor `runsc` RuntimeClass for kernel
 # isolation (uncomment `runtimeClassName` once the node has it installed).
+#
+# SINGLE TRUST DOMAIN: all sessions in this pod share one container/uid and are
+# isolated by working directory only (per-session cwd) — not by a kernel/OS
+# boundary. This image ships an env-scrubbing kernelspec under the distinct name
+# "docsgpt-python"; the app (docsgpt-api / docsgpt-worker) MUST select it via
+# SANDBOX_KERNEL_NAME=docsgpt-python (already set in docsgpt-deploy.yaml) so the
+# scrubber runs. The stock "python3" spec stays untouched, so it can never
+# shadow the scrubbing kernel. Sibling workspaces are still readable under the
+# shared uid and kernels share one address space. Do NOT inject app secrets
+# (DB/LLM creds) into this pod's env; the runner needs none. For cross-tenant /
+# untrusted multi-tenant workloads use a per-session VM via
+# SANDBOX_BACKEND=daytona instead.
 apiVersion: apps/v1
 kind: Deployment
 metadata:

deployment/sandbox/Dockerfile

@@ -42,6 +42,20 @@ RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
         pip install --no-cache-dir docling==2.8.3; \
     fi
 
+# Env-scrubbing kernel launcher + custom kernelspec. The launcher re-execs
+# ipykernel under a minimal allowlisted env (env -i) so NO secret in the
+# gateway's environment (*_API_KEY, *_TOKEN, POSTGRES_URI, the gateway auth
+# token, ...) ever reaches kernel code. The kernelspec ships under a DISTINCT
+# name ("docsgpt-python"), so the app selects it with SANDBOX_KERNEL_NAME and it
+# is never shadowed by the stock ipykernel "python3" spec regardless of the
+# python prefix. The stock "python3" spec is left untouched (no overwrite, no
+# kernelspec-name precedence to rely on). SECURITY: never give this image
+# `env_file: ../.env` -- the scrubber blocks exfil from the kernel, but the
+# runner image itself should stay free of app secrets it has no use for.
+COPY kernel-launch.sh /opt/docsgpt/kernel-launch.sh
+RUN chmod 0555 /opt/docsgpt/kernel-launch.sh
+COPY kernels/docsgpt-python/kernel.json /usr/local/share/jupyter/kernels/docsgpt-python/kernel.json
+
 USER sandbox
 WORKDIR /home/sandbox
 

deployment/sandbox/README.md

@@ -5,6 +5,58 @@ backend/worker is the **client** and connects over HTTP + WebSocket via
 `SANDBOX_GATEWAY_URL`. Each session is an **in-process kernel** (child process),
 never a child container; the Docker socket is **not** mounted.
 
+## Isolation model
+
+Read this before pointing untrusted or multi-tenant workloads at the runner.
+
+A single Jupyter runner is **one trust domain**. Every session is an in-process
+kernel under **one shared uid (10001)** in **one container**; sessions are
+isolated by **working directory only** — each session's code runs with its cwd
+set to its own `/tmp/docsgpt-sandbox/<session_id>` directory. That is a
+convenience boundary, not a security boundary between sessions.
+
+What this slice does close:
+
+- **Env-secret exfil is closed.** The custom kernelspec
+  (`kernels/docsgpt-python/kernel.json` → `/opt/docsgpt/kernel-launch.sh`)
+  re-execs ipykernel under a minimal allowlisted env (`env -i` keeping only
+  `PATH`, `HOME`, `LANG`, `JUPYTER_RUNTIME_DIR`, `JUPYTER_DATA_DIR`). The image
+  installs this spec under the **distinct name `docsgpt-python`** and the app
+  selects it via `SANDBOX_KERNEL_NAME=docsgpt-python`; because the name is
+  distinct, it is **never shadowed** by the stock ipykernel `python3` spec
+  (kernelspec name resolution prefers `sys.prefix/share` over
+  `/usr/local/share`, so reusing `python3` would silently fall back to the
+  unscrubbed stock spec on a different python prefix). The stock `python3` spec
+  is left untouched. So even though the gateway process inherits the operator's
+  full environment, **no `*_API_KEY` / `*_TOKEN` / `POSTGRES_URI` / gateway auth
+  token reaches kernel code** via `os.environ`, regardless of how the gateway is
+  launched. Loopback ZMQ reachability is preserved because `{connection_file}`
+  is forwarded untouched.
+- **Per-session workspace perms.** The workspace root and each session dir are
+  created `0700` (defense-in-depth). Under one shared uid this does **not** stop
+  a sibling session from reading another's files — it only narrows exposure to
+  other uids on the box.
+
+Residual gaps (treat all sessions in one runner as mutually trusting):
+
+- **Sibling-workspace reads.** All kernels run as the same uid, so one session's
+  code can read another session's files (and `/tmp`) despite `0700`. Distinct
+  uids / per-session VMs are required to close this.
+- **In-memory / cross-kernel.** Kernels are child processes of one gateway under
+  one uid; OS-level process isolation is the only boundary, and it is not a
+  sandbox boundary against a determined escape. No gVisor in the base posture.
+- **Egress.** Outbound is broad by design (so code can `pip install` / call
+  public APIs). Private/link-local/metadata ranges are blocked **only** by the
+  network layer — the k8s NetworkPolicy or a host/cloud firewall (see *Network
+  egress / SSRF* below), never by the runner itself.
+
+For real per-tenant isolation (cross-tenant or untrusted code), use the
+**Daytona backend** (`SANDBOX_BACKEND=daytona`), which gives each session its
+own VM. To harden the self-hosted Jupyter runner as a whole (host protection +
+egress), layer the **gVisor `runsc` runtime**, the **NetworkPolicy**, and a
+**host firewall** as documented below — those protect the host and constrain
+egress; they do **not** create a boundary between sessions inside one runner.
+
 ## Run standalone for dev
 
 Build and run the runner on its own, then point the app at it:
@@ -28,6 +80,16 @@ limit so large `get_file` base64 payloads aren't silently truncated. (On older
 gateways the trait may live elsewhere; the client's `get_file` integrity check
 catches any truncation regardless.)
 
+A bare-venv gateway uses the **stock** `python3` kernelspec, which inherits the
+gateway's full env (no secret scrubbing). The default `SANDBOX_KERNEL_NAME` is
+`python3`, so plain venv dev gets no scrubbing — acceptable for single-trust
+dev. The Docker image instead ships the env-scrubbing spec under the distinct
+name `docsgpt-python` (see *Isolation model*) and the runner stack sets
+`SANDBOX_KERNEL_NAME=docsgpt-python`. To get the scrubbing behavior in a venv,
+copy `kernels/docsgpt-python/kernel.json` (pointing `argv` at a local copy of
+`kernel-launch.sh`) into a Jupyter data dir on the kernelspec search path and
+set `SANDBOX_KERNEL_NAME=docsgpt-python` before launching.
+
 ## Exposing the port requires auth
 
 The image does **not** set `--KernelGatewayApp.allow_origin=*`. If you publish
@@ -41,7 +103,11 @@ required there.
 
 The `docsgpt-sandbox` service is defined in `deployment/docker-compose.yaml` on
 an internal-only network. The backend and worker reach it at
-`http://docsgpt-sandbox:8888`.
+`http://docsgpt-sandbox:8888` and select the scrubbing kernel by setting
+`SANDBOX_KERNEL_NAME=docsgpt-python` (the runner only ships the kernelspec; the
+app chooses it). The same applies to k8s: `SANDBOX_KERNEL_NAME=docsgpt-python`
+is set on the `docsgpt-api` and `docsgpt-worker` deployments in
+`deployment/k8s/deployments/docsgpt-deploy.yaml`.
 
 ## Document extraction variant (Docling)
 

deployment/sandbox/kernel-launch.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Env-scrubbing launcher for the docsgpt-sandbox ipykernel.
+#
+# The gateway process inherits the operator's full environment, which can carry
+# secrets (*_API_KEY, *_TOKEN, POSTGRES_URI, the gateway auth token, ...). Stock
+# kernels inherit that env verbatim, so LLM-authored code could read it via
+# os.environ. This wrapper re-execs ipykernel under a MINIMAL allowlisted env so
+# NO secret reaches kernel code, regardless of how the gateway was launched.
+#
+# Only what ipykernel needs is kept: PATH (find python), HOME (~/.ipython etc),
+# LANG (encoding), and the Jupyter runtime/data dirs (writable tmpfs paths). The
+# {connection_file} the gateway passes is forwarded via "$@" so loopback ZMQ
+# reachability is preserved -- do NOT drop or rewrite those args.
+exec env -i \
+    PATH="${PATH}" \
+    HOME="${HOME}" \
+    LANG="${LANG}" \
+    JUPYTER_RUNTIME_DIR="${JUPYTER_RUNTIME_DIR}" \
+    JUPYTER_DATA_DIR="${JUPYTER_DATA_DIR}" \
+    python -m ipykernel_launcher "$@"

deployment/sandbox/kernels/docsgpt-python/kernel.json

@@ -0,0 +1,12 @@
+{
+  "argv": [
+    "/opt/docsgpt/kernel-launch.sh",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "Python 3 (docsgpt-sandbox, scrubbed env)",
+  "language": "python",
+  "metadata": {
+    "debugger": true
+  }
+}

tests/sandbox/test_jupyter_gateway_isolation.py

@@ -0,0 +1,147 @@
+"""Per-session isolation hardening for the Jupyter gateway sandbox.
+
+Covers the env-scrubbing kernel launcher (`deployment/sandbox/kernel-launch.sh`)
+and the `0700` per-session workspace perms applied by `_prime`. Hermetic: the
+launcher test runs the wrapper with a fake `python` on PATH, and the `_prime`
+test executes the wrapper's setup code against a real temp directory by
+stubbing `_run` -- no gateway / kernel process required.
+"""
+
+import json
+import os
+import shutil
+import stat
+import subprocess
+import textwrap
+from pathlib import Path
+
+import pytest
+
+from application.sandbox import jupyter_gateway
+from application.sandbox.base import ExecResult
+from application.sandbox.jupyter_gateway import JupyterKernelGatewaySandbox, _Kernel
+
+_SANDBOX_DIR = Path(__file__).resolve().parents[2] / "deployment" / "sandbox"
+_WRAPPER = _SANDBOX_DIR / "kernel-launch.sh"
+_KERNEL_NAME = "docsgpt-python"
+_KERNELSPEC = _SANDBOX_DIR / "kernels" / _KERNEL_NAME / "kernel.json"
+
+
+# -- Env-scrubbing kernel launcher ---------------------------------------------
+
+
+@pytest.mark.skipif(shutil.which("sh") is None, reason="POSIX sh not available")
+def test_kernel_launch_scrubs_secrets_keeps_runtime_env(tmp_path):
+    """The wrapper drops *_API_KEY/*_TOKEN but keeps PATH/HOME/JUPYTER_* for ipykernel."""
+    # Fake `python` on PATH: ignore `-m ipykernel_launcher` and dump the env it was given.
+    fake_python = tmp_path / "python"
+    fake_python.write_text(
+        textwrap.dedent(
+            """\
+            #!/bin/sh
+            env
+            """
+        )
+    )
+    fake_python.chmod(0o755)
+
+    env = {
+        "PATH": f"{tmp_path}:{os.environ.get('PATH', '')}",
+        "HOME": str(tmp_path),
+        "LANG": "C.UTF-8",
+        "JUPYTER_RUNTIME_DIR": str(tmp_path / "runtime"),
+        "JUPYTER_DATA_DIR": str(tmp_path / "data"),
+        # Secrets that must NOT reach the kernel.
+        "OPENAI_API_KEY": "sk-super-secret",
+        "SANDBOX_GATEWAY_AUTH_TOKEN": "gateway-token",
+        "POSTGRES_URI": "postgresql://u:p@h/db",
+    }
+    proc = subprocess.run(
+        ["sh", str(_WRAPPER), "-f", "/tmp/conn.json"],
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=15,
+    )
+    assert proc.returncode == 0, proc.stderr
+    out = proc.stdout
+    # Secrets stripped.
+    assert "OPENAI_API_KEY" not in out
+    assert "sk-super-secret" not in out
+    assert "SANDBOX_GATEWAY_AUTH_TOKEN" not in out
+    assert "POSTGRES_URI" not in out
+    # Allowlisted runtime env kept.
+    assert "PATH=" in out
+    assert f"HOME={tmp_path}" in out
+    assert f"JUPYTER_RUNTIME_DIR={tmp_path / 'runtime'}" in out
+    assert f"JUPYTER_DATA_DIR={tmp_path / 'data'}" in out
+    # The connection-file args were forwarded to ipykernel (reachability preserved).
+    # The fake python prints env only, so just assert it was invoked with no crash above.
+
+
+@pytest.mark.skipif(shutil.which("sh") is None, reason="POSIX sh not available")
+def test_kernel_launch_is_valid_sh():
+    """The wrapper parses under POSIX sh (`sh -n`)."""
+    proc = subprocess.run(["sh", "-n", str(_WRAPPER)], capture_output=True, text=True)
+    assert proc.returncode == 0, proc.stderr
+
+
+# -- Scrubbing kernelspec is selectable ----------------------------------------
+
+
+def test_kernelspec_argv_points_at_scrubbing_wrapper():
+    """The shipped kernel.json launches the env-scrubbing wrapper, not bare ipykernel."""
+    spec = json.loads(_KERNELSPEC.read_text())
+    argv = spec["argv"]
+    assert argv[0].endswith("kernel-launch.sh")
+    assert "{connection_file}" in argv
+
+
+def test_distinct_kernel_name_resolves_to_scrubbing_spec(tmp_path, monkeypatch):
+    """A distinct kernel name resolves to the scrubbing wrapper (never the stock python3 spec)."""
+    kernelspec = pytest.importorskip("jupyter_client.kernelspec")
+
+    # Seed a Jupyter data dir with the custom spec under its distinct name.
+    data_dir = tmp_path / "jupyter"
+    spec_dir = data_dir / "kernels" / _KERNEL_NAME
+    spec_dir.mkdir(parents=True)
+    shutil.copy(_KERNELSPEC, spec_dir / "kernel.json")
+    monkeypatch.setenv("JUPYTER_PATH", str(data_dir))
+
+    manager = kernelspec.KernelSpecManager()
+    resolved = manager.get_kernel_spec(_KERNEL_NAME)
+    assert resolved.argv[0].endswith("kernel-launch.sh")
+    assert "{connection_file}" in resolved.argv
+
+
+# -- Per-session workspace perms (0700) ----------------------------------------
+
+
+def _exec_setup_in_tmp(code: str) -> None:
+    """Run the kernel-side setup snippet in-process (it is plain os.* calls)."""
+    exec(compile(code, "<setup>", "exec"), {})
+
+
+def test_prime_creates_workspace_mode_0700(tmp_path, monkeypatch):
+    """`_prime` creates the workspace root and per-session dir at mode 0700."""
+    root = tmp_path / "docsgpt-sandbox"
+    monkeypatch.setattr(jupyter_gateway, "_WORKSPACE_ROOT", str(root))
+
+    sb = JupyterKernelGatewaySandbox(gateway_url="http://unused")
+    workspace = f"{root}/conv-perms"
+    kernel = _Kernel("kid", workspace)
+
+    captured = {}
+
+    def fake_run(_kernel, code, _timeout):
+        captured["code"] = code
+        _exec_setup_in_tmp(code)
+        return ExecResult(status="ok", exit_code=0)
+
+    monkeypatch.setattr(sb, "_run", fake_run)
+    monkeypatch.chdir(tmp_path)  # _prime's os.chdir must land somewhere harmless
+    sb._prime(kernel)
+
+    assert kernel.initialized
+    assert stat.S_IMODE(os.stat(root).st_mode) == 0o700
+    assert stat.S_IMODE(os.stat(workspace).st_mode) == 0o700