mirror of
https://github.com/tiennm99/miti99bot-js.git
synced 2026-05-14 19:53:04 +00:00
0859356ec7
Operator-run migration scripts for KV→Mongo and D1→trading_trades, plus a parity verifier and a rollback wiper. Pure local Node — no Worker code, no /__admin/* routes, no new Worker secrets. Complies with docs/architecture.md §10. Scripts - backfill-kv-to-mongo.js: paginates CF KV REST API per module, fetches values, $setOnInsert upsert into per-module Mongo collection. Resumes from .backfill-cursor-<module>.json on restart. Throttles 50 ops/sec. expiresAt derived from KV metadata.expiration (debugger #10). --dry-run and --module flags for incremental work. - backfill-d1-to-mongo.js: wrangler d1 execute --remote --json → parse → insertMany batches into trading_trades, preserving original integer id as legacy_id (code-reviewer #13). Pre-flight aborts if collection non-empty unless --force. - verify-mongo-parity.js: count parity ±1%, SHA256 value compare, expiresAt ±5min bucket. Full-scan when <10K docs, sqrt-sample capped at 500 otherwise (code-reviewer #21). Trading: full-scan on legacy_id/ts/user_id/symbol/qty. - wipe-mongo.js: rollback helper. deleteMany across all collections with readline confirm. --yes for CI. - lib/migration-helpers.js: shared sleep, sha256, checkpoint I/O, cfKvList/cfKvGet, MongoClient singleton, sample strategy. Surface updates - .env.deploy.example: CF account/token/namespace placeholders. - package.json: backfill:kv[:dry], backfill:d1[:dry], verify:mongo, wipe:mongo scripts. - check-secret-leaks.js: SECRETS array gains CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID for defense-in-depth. - .gitignore: .backfill-cursor-*.json excluded. Tests: 638 → 667 (+29 pure-logic tests for sha256, checkpoint round-trip, count-diff, sample-size, fetch-mocked CF REST). Lint clean. Operator-run sequence (after Phase 06 deploy): npm run backfill:kv:dry # preview npm run backfill:kv npm run backfill:d1:dry npm run backfill:d1 npm run verify:mongo # exit 0 = parity ok
95 lines
3.4 KiB
JavaScript
95 lines
3.4 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* @file wipe-mongo — rollback helper: delete all documents from every backfill
|
|
* collection in MongoDB Atlas.
|
|
*
|
|
* WARNING: THIS IS IRREVERSIBLE. Run only when you want to start the backfill
|
|
* from scratch (e.g. after discovering a systematic mapping bug).
|
|
*
|
|
* Flags:
|
|
* --yes Skip the interactive confirmation prompt (for CI / scripted rollback).
|
|
* Even with --yes, prints a loud warning so logs capture intent.
|
|
*
|
|
* Required env: MONGODB_URI, MODULES (comma-separated KV module names)
|
|
*
|
|
* Usage:
|
|
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js
|
|
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js --yes
|
|
*/
|
|
|
|
import { createInterface } from "node:readline";
|
|
import { closeMongoClient, getMongoClient } from "./lib/migration-helpers.js";
|
|
|
|
const { MONGODB_URI, MODULES: MODULES_ENV } = process.env;
|
|
const skipPrompt = process.argv.includes("--yes");
|
|
|
|
function validateEnv() {
|
|
const needed = { MONGODB_URI, MODULES: MODULES_ENV };
|
|
const missing = Object.entries(needed)
|
|
.filter(([, v]) => !v)
|
|
.map(([k]) => k);
|
|
if (missing.length) {
|
|
console.error(`[wipe-mongo] Missing required env vars: ${missing.join(", ")}`);
|
|
console.error(" Copy .env.deploy.example to .env.deploy and fill in values.");
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
/** @param {string} question @returns {Promise<string>} */
|
|
function prompt(question) {
|
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
return new Promise((resolve) => {
|
|
rl.question(question, (answer) => {
|
|
rl.close();
|
|
resolve(answer);
|
|
});
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
validateEnv();
|
|
|
|
// Build the list of collections: one per KV module + trading_trades.
|
|
const kvModules = MODULES_ENV.split(",")
|
|
.map((m) => m.trim())
|
|
.filter(Boolean);
|
|
const collections = [
|
|
...kvModules.map((m) => m.replace(/-/g, "_")), // mirrors mongo-kv-store.js normalization
|
|
"trading_trades",
|
|
];
|
|
|
|
console.error("╔══════════════════════════════════════════════════════╗");
|
|
console.error("║ WARNING: wipe-mongo will DELETE ALL DOCUMENTS from ║");
|
|
console.error("║ the Atlas database and CANNOT BE UNDONE. ║");
|
|
console.error("╚══════════════════════════════════════════════════════╝");
|
|
console.log(`Collections to wipe (${collections.length}): ${collections.join(", ")}`);
|
|
|
|
if (!skipPrompt) {
|
|
const answer = await prompt("\nType CONFIRM to wipe Atlas database miti99bot: ");
|
|
if (answer.trim() !== "CONFIRM") {
|
|
console.log("Aborted — nothing was deleted.");
|
|
process.exit(0);
|
|
}
|
|
} else {
|
|
console.error("[wipe-mongo] --yes passed: skipping interactive prompt. Proceeding with wipe.");
|
|
}
|
|
|
|
const client = await getMongoClient(MONGODB_URI);
|
|
const db = client.db();
|
|
|
|
let totalDeleted = 0;
|
|
for (const name of collections) {
|
|
const result = await db.collection(name).deleteMany({});
|
|
console.log(`[wipe-mongo] ${name}: deleted ${result.deletedCount} document(s)`);
|
|
totalDeleted += result.deletedCount;
|
|
}
|
|
|
|
await closeMongoClient();
|
|
console.log(`[wipe-mongo] Done — ${totalDeleted} total document(s) removed.`);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("[wipe-mongo] Fatal:", err.message ?? err);
|
|
process.exit(1);
|
|
});
|