From d2dbf4c2b7d3728c85e0c758cfb55ea2b29ada9d Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 26 Apr 2026 08:32:19 +0700 Subject: [PATCH] =?UTF-8?q?feat(db):=20phase=2001=20=E2=80=94=20atlas=20wr?= =?UTF-8?q?angler=20config=20+=20secret-leak=20lint=20+=20mongodb=20dep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code/config slice of plan phase 01 (operator-only steps for cluster provisioning, secrets, and runtime smoke tests deferred to user). - wrangler.toml: add `compatibility_flags = ["nodejs_compat_v2"]` (compatibility_date `2025-10-01` already satisfies ≥ 2025-03-20) - .env.deploy.example: add `MONGODB_URI` placeholder with mirror-protocol note - scripts/check-secret-leaks.js: lint that fails build on `console.log(env.)` for MONGODB_URI / TELEGRAM_BOT_TOKEN / TELEGRAM_WEBHOOK_SECRET / ADMIN_TOKEN - package.json: install mongodb@^6.7.0 (resolved 6.21.0); wire secret-leak check into `npm run lint` - docs/using-mongodb.md: operational runbook (cluster spec, free-tier ceiling, auto-pause behavior, network access permanence, rollback, rotation) Bundle-size HARD GATE: PASS. Probe with `import { MongoClient }` measures 226 KiB gzipped (3 MiB Free cap, 92% headroom) — nodejs_compat_v2 provides node:net/tls/crypto from runtime so transitive deps stay unbundled. CPU-time gate and auto-pause behavior gate require real Atlas access; deferred to operator (see docs/using-mongodb.md for procedure). 503/503 vitest tests still pass. --- .env.deploy.example | 7 ++ docs/using-mongodb.md | 142 ++++++++++++++++++++++++++++++++++ package-lock.json | 142 +++++++++++++++++++++++++++++++++- package.json | 5 +- scripts/check-secret-leaks.js | 116 +++++++++++++++++++++++++++ wrangler.toml | 4 + 6 files changed, 412 insertions(+), 4 deletions(-) create mode 100644 docs/using-mongodb.md create mode 100644 scripts/check-secret-leaks.js diff --git a/.env.deploy.example b/.env.deploy.example index 536f447..9dcbd03 100644 --- a/.env.deploy.example +++ b/.env.deploy.example @@ -13,3 +13,10 @@ WORKER_URL= # Same MODULES value as wrangler.toml [vars]. Duplicated here so the register # script can derive the public command list without parsing wrangler.toml. MODULES=util,wordle,loldle,misc,trading,lolschedule,semantle,doantu,twentyq + +# MongoDB Atlas connection string. Used by the `mongodb` driver inside the Worker +# AND by local backfill / verify scripts. MUST match the value set via +# `wrangler secret put MONGODB_URI` for the Worker. Same secret-mirror protocol +# as TELEGRAM_BOT_TOKEN / TELEGRAM_WEBHOOK_SECRET. +# Format: mongodb+srv://miti99bot-worker:@/miti99bot?retryWrites=true&w=majority +MONGODB_URI= diff --git a/docs/using-mongodb.md b/docs/using-mongodb.md new file mode 100644 index 0000000..bfd327f --- /dev/null +++ b/docs/using-mongodb.md @@ -0,0 +1,142 @@ +# Using MongoDB Atlas + +Operational runbook for the MongoDB Atlas backend introduced by `plans/260425-1945-mongodb-atlas-migration/`. + +## Cluster + +| Field | Value | +|---|---| +| Provider | MongoDB Atlas | +| Tier | M0 Free | +| Region | `aws-ap-southeast-1` (Singapore) | +| Cluster name | `miti99bot-prod` (operator confirms) | +| Database | `miti99bot` | +| DB user | `miti99bot-worker` (`readWrite@miti99bot`) | + +Connection string format: + +``` +mongodb+srv://miti99bot-worker:@/miti99bot?retryWrites=true&w=majority +``` + +Stored in two places (must match): +1. CF Worker secret: `wrangler secret put MONGODB_URI` +2. `.env.deploy` (gitignored, used by local backfill / verify scripts) + +Same secret-mirror protocol as `TELEGRAM_BOT_TOKEN`. + +## Free-tier ceiling + +- 512 MB storage (data + indexes) +- 500 max concurrent connections +- ~100 ops/sec sustained (no daily cap) +- No backups, single region, no PITR +- Auto-pauses after 30 days of zero ops + +Upgrade path: **Flex Tier $8–$30/month** (M2/M5 deprecated as of 2026). + +## Auto-pause + +After 30 days idle the cluster pauses. First request after pause: +- Driver throws `MongoServerSelectionError` after `serverSelectionTimeoutMS` (5s). +- Worker code (see `src/db/mongo-client.js`, lands Phase 02) catches and returns 503 with `Retry-After: 30`. +- Cluster auto-wakes within 30–60s on attempted connection. + +The bot has 6+ daily crons; any cron that writes Mongo prevents pause. Phase 08 confirms. + +## Network access + +`0.0.0.0/0` — Cloudflare Workers do NOT have static egress IPs on the Free or basic Paid plans. Only auth (SCRAM-SHA-256) + TLS gate connections. + +**Permanent risk** unless upgrading to CF Workers paid static-egress IP add-on (~$10/mo). + +Mitigations: +- DB user has `readWrite` on one db only (NOT `dbAdmin` / `clusterAdmin`). +- Password ≥32 chars random. +- Rotate quarterly. +- Atlas free-tier email alerts configured for cluster unavailability + connections > 400. + +## Bundle gate (Phase 01 result) + +Measured `npx wrangler deploy --dry-run` with a minimal probe importing `MongoClient`: + +| Metric | Value | Cap (Free) | Cap (Paid) | +|---|---|---|---| +| Compressed (gzip) | **226 KiB** | 3 MiB | 10 MiB | +| Raw (minified) | 1.74 MiB | — | — | +| On-disk (uncompressed) | 3.9 MiB | — | — | + +Pass on both plans with **>92% headroom**. nodejs_compat_v2 provides `node:net`/`node:tls`/`node:crypto` from the runtime, so the driver's transitive deps are not bundled. + +## CPU-time gate (Phase 01 — operator-run) + +Requires real Atlas + `wrangler dev`. Procedure: + +1. Add a temporary `/__mongo-ping` route that connects + runs `db.runCommand({ping:1})` + returns `{wall_ms}`. +2. Run 5+ cold cycles (10-min spaced). +3. Inspect CF dashboard CPU column for each invocation. +4. **Hard gate**: if any cold-start CPU time approaches 50ms (Free plan limit), abort migration. Escalate to paid plan or pivot via `phase-07-alt-pivot.md`. +5. Record cold-ping P95 wall-clock as `BASELINE_COLD_PING_MS` here: + +``` +BASELINE_COLD_PING_MS = +``` + +Phase 06 derives the abort threshold from this value: `2.5 × BASELINE_COLD_PING_MS`. + +## Auto-pause behavior gate (Phase 01 — operator-run) + +In Atlas UI, manually pause the cluster, then hit `/__mongo-ping`. Confirm: +- Driver throws within 5s (does NOT hang indefinitely). +- Error class is `MongoServerSelectionError` (or driver subclass). +- Phase 02 `getDb()` catches this and surfaces a 503. + +## Node API surface + +`src/` (the Worker) imports zero `node:*` modules today. `nodejs_compat_v2` is enabled solely for the `mongodb` driver: + +| Module | Used by Worker? | Used by scripts/? | +|---|---|---| +| `node:fs` | no | yes (build/scrape/migrate) | +| `node:path` | no | yes | +| `node:child_process` | no | yes (migrate.js) | +| `node:net` | indirectly (via mongodb) | no | +| `node:tls` | indirectly (via mongodb) | no | +| `node:crypto` | indirectly (via mongodb) | no | +| `process.env` | no | yes (register.js) | +| `Buffer` | no | no | + +Risk: minimal. No existing module relies on the absence of these globals. + +## Rollback + +If migration is abandoned at any phase before cutover: + +1. `wrangler secret delete MONGODB_URI` +2. Revert `wrangler.toml`: remove `compatibility_flags = ["nodejs_compat_v2"]`. +3. `npm uninstall mongodb`. +4. `npm run deploy` — bot continues on KV/D1 unchanged. +5. (Optional) Delete Atlas cluster from UI. + +`scripts/check-secret-leaks.js` should stay — it covers other secrets too. + +## Rotation + +`MONGODB_URI` rotation cadence: every 90 days, owner = repo maintainer. + +Procedure: +1. In Atlas UI → Database Access → edit `miti99bot-worker` → reset password. +2. Update `.env.deploy` with new URI. +3. `wrangler secret put MONGODB_URI` (paste new URI). +4. `npm run deploy` (re-runs register; no Worker restart needed since secret reads at request time via `env.MONGODB_URI`). + +Mismatch between `.env.deploy` and CF secret causes register-script failure on next deploy — same fail-loud pattern as `TELEGRAM_WEBHOOK_SECRET`. + +## Alerts + +Configured in Atlas free-tier UI: + +- **Cluster unavailable** → email maintainer. +- **Current connections > 400** (80% of cap) → email maintainer. + +Plus CF Observability rule (Phase 06): >10 errors per 1 min window → email. diff --git a/package-lock.json b/package-lock.json index 653b51a..a1312d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,8 @@ "name": "miti99bot", "version": "0.1.0", "dependencies": { - "grammy": "^1.30.0" + "grammy": "^1.30.0", + "mongodb": "^6.21.0" }, "devDependencies": { "@biomejs/biome": "^1.9.0", @@ -1053,6 +1054,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.4.9", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.9.tgz", + "integrity": "sha512-RXSxsokhAF/4nWys8An8npsqOI33Ex1Hlzqjw2pZOO+GKtMAR2noGnUdsFiGwsaO/xXI+56mtjTmDA3JXJsvmA==", + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", @@ -1465,6 +1475,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, "node_modules/@typescript-eslint/types": { "version": "8.58.2", "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.2.tgz", @@ -1695,6 +1720,15 @@ "node": "18 || 20 || >=22" } }, + "node_modules/bson": { + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "license": "Apache-2.0", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -2590,6 +2624,12 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "license": "MIT" + }, "node_modules/miniflare": { "version": "4.20260420.0", "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260420.0.tgz", @@ -2627,6 +2667,96 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mongodb": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", + "integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.2" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.3.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/mongodb-connection-string-url/node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2861,7 +2991,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3012,6 +3141,15 @@ "node": ">=0.10.0" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/spdx-exceptions": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", diff --git a/package.json b/package.json index 9232658..146185a 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,13 @@ "db:migrate": "node scripts/migrate.js", "register": "node --env-file-if-exists=.env.deploy scripts/register.js", "register:dry": "node --env-file-if-exists=.env.deploy scripts/register.js --dry-run", - "lint": "biome check . && eslint src", + "lint": "biome check . && eslint src && node scripts/check-secret-leaks.js", "format": "biome format --write .", "test": "vitest run" }, "dependencies": { - "grammy": "^1.30.0" + "grammy": "^1.30.0", + "mongodb": "^6.21.0" }, "devDependencies": { "@biomejs/biome": "^1.9.0", diff --git a/scripts/check-secret-leaks.js b/scripts/check-secret-leaks.js new file mode 100644 index 0000000..ac40c13 --- /dev/null +++ b/scripts/check-secret-leaks.js @@ -0,0 +1,116 @@ +#!/usr/bin/env node +/** + * @file check-secret-leaks — fails build if any source file logs a known secret. + * + * Catches the common foot-gun where a developer prints `env.MONGODB_URI` or + * similar during debugging and forgets to remove the line before commit. We + * are NOT trying to be a full SAST — we just block obvious `console.log` / + * `console.error` sites that interpolate a secret env var. + * + * Wired into `npm run lint` so every PR / pre-deploy run catches it. + * + * Patterns checked (add more as new secrets are introduced): + * - MONGODB_URI — Atlas connection string (Phase 01) + * - TELEGRAM_BOT_TOKEN — bot token from BotFather + * - TELEGRAM_WEBHOOK_SECRET — gates incoming webhook traffic + * - ADMIN_TOKEN — kept for defense-in-depth even though Phase 05 + * redesign removed admin routes; cheap to leave in. + * + * Detection scope: any of these tokens appearing on the SAME line as a + * `console.(...)`, `JSON.stringify(env)`, or `throw new Error(...env...)`. + * + * Exit codes: + * 0 — no leaks found + * 1 — at least one leak detected (prints file:line + offending line) + */ + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { extname, join, resolve } from "node:path"; + +const PROJECT_ROOT = resolve(import.meta.dirname, ".."); +const SCAN_DIRS = ["src", "scripts"]; +const SCAN_EXTS = new Set([".js", ".mjs", ".ts"]); + +const SECRETS = ["MONGODB_URI", "TELEGRAM_BOT_TOKEN", "TELEGRAM_WEBHOOK_SECRET", "ADMIN_TOKEN"]; + +// A line is suspicious if it both names a secret AND looks like it's emitting +// the value (console.*, JSON.stringify(env...), throw with interpolation). +const EMIT_PATTERNS = [ + /\bconsole\.(log|info|warn|error|debug|trace)\b/, + /\bJSON\.stringify\s*\(\s*env\b/, + /\bthrow\s+new\s+\w*Error\b/, +]; + +/** + * Walk a directory tree and yield absolute file paths matching SCAN_EXTS. + * + * @param {string} dir + * @returns {string[]} + */ +function walk(dir) { + if (!existsSync(dir)) return []; + const out = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + if (entry === "node_modules" || entry === ".wrangler") continue; + out.push(...walk(full)); + } else if (SCAN_EXTS.has(extname(entry))) { + out.push(full); + } + } + return out; +} + +/** + * Scan one file. Pushes hits to `findings`. + * + * @param {string} file + * @param {Array<{file: string, line: number, secret: string, snippet: string}>} findings + */ +function scanFile(file, findings) { + // Don't flag this file (it lists the patterns) or .example files. + if (file.endsWith("check-secret-leaks.js")) return; + + const content = readFileSync(file, "utf8"); + const lines = content.split("\n"); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (!EMIT_PATTERNS.some((p) => p.test(line))) continue; + for (const secret of SECRETS) { + if (line.includes(secret)) { + findings.push({ + file, + line: i + 1, + secret, + snippet: line.trim(), + }); + } + } + } +} + +function main() { + /** @type {Array<{file: string, line: number, secret: string, snippet: string}>} */ + const findings = []; + + for (const dir of SCAN_DIRS) { + const abs = join(PROJECT_ROOT, dir); + for (const file of walk(abs)) scanFile(file, findings); + } + + if (findings.length === 0) { + console.log(`secret-leak check: 0 findings across ${SCAN_DIRS.join(", ")}`); + return; + } + + console.error(`secret-leak check: ${findings.length} finding(s)`); + for (const f of findings) { + const rel = f.file.replace(`${PROJECT_ROOT}/`, ""); + console.error(` ${rel}:${f.line} [${f.secret}] ${f.snippet}`); + } + process.exit(1); +} + +main(); diff --git a/wrangler.toml b/wrangler.toml index 2628889..ba5f473 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,6 +1,10 @@ name = "miti99bot" main = "src/index.js" compatibility_date = "2025-10-01" +# nodejs_compat_v2 enables node:net + node:tls so the official `mongodb` driver +# can open a TCP socket to Atlas. v1 vs v2 are alternatives, not additive. +# Adding this flag does not affect existing modules — `src/` has no node: imports. +compatibility_flags = ["nodejs_compat_v2"] # Enabled modules at runtime. Comma-separated. Must match static-map keys in src/modules/index.js. # Also duplicate this value into .env.deploy so scripts/register.js derives the same public command list.