Files
tiennm99 0859356ec7 feat(scripts): phase 05 — backfill + verify + wipe (local node, no admin routes)
Operator-run migration scripts for KV→Mongo and D1→trading_trades, plus a
parity verifier and a rollback wiper. Pure local Node — no Worker code,
no /__admin/* routes, no new Worker secrets. Complies with
docs/architecture.md §10.

Scripts
- backfill-kv-to-mongo.js: paginates CF KV REST API per module, fetches
  values, $setOnInsert upsert into per-module Mongo collection. Resumes
  from .backfill-cursor-<module>.json on restart. Throttles 50 ops/sec.
  expiresAt derived from KV metadata.expiration (debugger #10). --dry-run
  and --module flags for incremental work.
- backfill-d1-to-mongo.js: wrangler d1 execute --remote --json → parse →
  insertMany batches into trading_trades, preserving original integer id
  as legacy_id (code-reviewer #13). Pre-flight aborts if collection
  non-empty unless --force.
- verify-mongo-parity.js: count parity ±1%, SHA256 value compare,
  expiresAt ±5min bucket. Full-scan when <10K docs, sqrt-sample
  capped at 500 otherwise (code-reviewer #21). Trading: full-scan
  on legacy_id/ts/user_id/symbol/qty.
- wipe-mongo.js: rollback helper. deleteMany across all collections
  with readline confirm. --yes for CI.
- lib/migration-helpers.js: shared sleep, sha256, checkpoint I/O,
  cfKvList/cfKvGet, MongoClient singleton, sample strategy.

Surface updates
- .env.deploy.example: CF account/token/namespace placeholders.
- package.json: backfill:kv[:dry], backfill:d1[:dry], verify:mongo,
  wipe:mongo scripts.
- check-secret-leaks.js: SECRETS array gains CLOUDFLARE_API_TOKEN +
  CLOUDFLARE_ACCOUNT_ID for defense-in-depth.
- .gitignore: .backfill-cursor-*.json excluded.

Tests: 638 → 667 (+29 pure-logic tests for sha256, checkpoint round-trip,
count-diff, sample-size, fetch-mocked CF REST). Lint clean.

Operator-run sequence (after Phase 06 deploy):
  npm run backfill:kv:dry   # preview
  npm run backfill:kv
  npm run backfill:d1:dry
  npm run backfill:d1
  npm run verify:mongo      # exit 0 = parity ok
2026-04-26 09:13:00 +07:00

95 lines
3.4 KiB
JavaScript

#!/usr/bin/env node
/**
* @file wipe-mongo — rollback helper: delete all documents from every backfill
* collection in MongoDB Atlas.
*
* WARNING: THIS IS IRREVERSIBLE. Run only when you want to start the backfill
* from scratch (e.g. after discovering a systematic mapping bug).
*
* Flags:
* --yes Skip the interactive confirmation prompt (for CI / scripted rollback).
* Even with --yes, prints a loud warning so logs capture intent.
*
* Required env: MONGODB_URI, MODULES (comma-separated KV module names)
*
* Usage:
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js --yes
*/
import { createInterface } from "node:readline";
import { closeMongoClient, getMongoClient } from "./lib/migration-helpers.js";
const { MONGODB_URI, MODULES: MODULES_ENV } = process.env;
const skipPrompt = process.argv.includes("--yes");
function validateEnv() {
const needed = { MONGODB_URI, MODULES: MODULES_ENV };
const missing = Object.entries(needed)
.filter(([, v]) => !v)
.map(([k]) => k);
if (missing.length) {
console.error(`[wipe-mongo] Missing required env vars: ${missing.join(", ")}`);
console.error(" Copy .env.deploy.example to .env.deploy and fill in values.");
process.exit(1);
}
}
/** @param {string} question @returns {Promise<string>} */
function prompt(question) {
const rl = createInterface({ input: process.stdin, output: process.stdout });
return new Promise((resolve) => {
rl.question(question, (answer) => {
rl.close();
resolve(answer);
});
});
}
async function main() {
validateEnv();
// Build the list of collections: one per KV module + trading_trades.
const kvModules = MODULES_ENV.split(",")
.map((m) => m.trim())
.filter(Boolean);
const collections = [
...kvModules.map((m) => m.replace(/-/g, "_")), // mirrors mongo-kv-store.js normalization
"trading_trades",
];
console.error("╔══════════════════════════════════════════════════════╗");
console.error("║ WARNING: wipe-mongo will DELETE ALL DOCUMENTS from ║");
console.error("║ the Atlas database and CANNOT BE UNDONE. ║");
console.error("╚══════════════════════════════════════════════════════╝");
console.log(`Collections to wipe (${collections.length}): ${collections.join(", ")}`);
if (!skipPrompt) {
const answer = await prompt("\nType CONFIRM to wipe Atlas database miti99bot: ");
if (answer.trim() !== "CONFIRM") {
console.log("Aborted — nothing was deleted.");
process.exit(0);
}
} else {
console.error("[wipe-mongo] --yes passed: skipping interactive prompt. Proceeding with wipe.");
}
const client = await getMongoClient(MONGODB_URI);
const db = client.db();
let totalDeleted = 0;
for (const name of collections) {
const result = await db.collection(name).deleteMany({});
console.log(`[wipe-mongo] ${name}: deleted ${result.deletedCount} document(s)`);
totalDeleted += result.deletedCount;
}
await closeMongoClient();
console.log(`[wipe-mongo] Done — ${totalDeleted} total document(s) removed.`);
}
main().catch((err) => {
console.error("[wipe-mongo] Fatal:", err.message ?? err);
process.exit(1);
});