mirror of
https://github.com/tiennm99/miti99bot.git
synced 2026-09-19 04:22:39 +00:00
Operator-run migration scripts for KV→Mongo and D1→trading_trades, plus a parity verifier and a rollback wiper. Pure local Node — no Worker code, no /__admin/* routes, no new Worker secrets. Complies with docs/architecture.md §10. Scripts - backfill-kv-to-mongo.js: paginates CF KV REST API per module, fetches values, $setOnInsert upsert into per-module Mongo collection. Resumes from .backfill-cursor-<module>.json on restart. Throttles 50 ops/sec. expiresAt derived from KV metadata.expiration (debugger #10). --dry-run and --module flags for incremental work. - backfill-d1-to-mongo.js: wrangler d1 execute --remote --json → parse → insertMany batches into trading_trades, preserving original integer id as legacy_id (code-reviewer #13). Pre-flight aborts if collection non-empty unless --force. - verify-mongo-parity.js: count parity ±1%, SHA256 value compare, expiresAt ±5min bucket. Full-scan when <10K docs, sqrt-sample capped at 500 otherwise (code-reviewer #21). Trading: full-scan on legacy_id/ts/user_id/symbol/qty. - wipe-mongo.js: rollback helper. deleteMany across all collections with readline confirm. --yes for CI. - lib/migration-helpers.js: shared sleep, sha256, checkpoint I/O, cfKvList/cfKvGet, MongoClient singleton, sample strategy. Surface updates - .env.deploy.example: CF account/token/namespace placeholders. - package.json: backfill:kv[:dry], backfill:d1[:dry], verify:mongo, wipe:mongo scripts. - check-secret-leaks.js: SECRETS array gains CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID for defense-in-depth. - .gitignore: .backfill-cursor-*.json excluded. Tests: 638 → 667 (+29 pure-logic tests for sha256, checkpoint round-trip, count-diff, sample-size, fetch-mocked CF REST). Lint clean. Operator-run sequence (after Phase 06 deploy): npm run backfill:kv:dry # preview npm run backfill:kv npm run backfill:d1:dry npm run backfill:d1 npm run verify:mongo # exit 0 = parity ok
266 lines
9.0 KiB
JavaScript
266 lines
9.0 KiB
JavaScript
/**
|
|
* @file migration-helpers.test.js — pure-logic unit tests for migration-helpers.
|
|
*
|
|
* Tests: sha256, checkpoint round-trip, countDiffRatio, sampleStrategy,
|
|
* cfKvList / cfKvGet response parsing (via fetch mock).
|
|
*
|
|
* No real Atlas connection, no real CF REST calls.
|
|
*/
|
|
|
|
import { existsSync, unlinkSync, writeFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
cfKvGet,
|
|
cfKvList,
|
|
clearCheckpoint,
|
|
closeMongoClient,
|
|
countDiffRatio,
|
|
loadCheckpoint,
|
|
sampleStrategy,
|
|
saveCheckpoint,
|
|
sha256,
|
|
sleep,
|
|
} from "../../scripts/lib/migration-helpers.js";
|
|
|
|
// ─── sleep ────────────────────────────────────────────────────────────────────
|
|
|
|
describe("sleep", () => {
|
|
it("resolves after at least ms milliseconds", async () => {
|
|
const start = Date.now();
|
|
await sleep(20);
|
|
expect(Date.now() - start).toBeGreaterThanOrEqual(15); // allow 5ms jitter
|
|
});
|
|
});
|
|
|
|
// ─── sha256 ───────────────────────────────────────────────────────────────────
|
|
|
|
describe("sha256", () => {
|
|
it("produces a 64-char hex string", () => {
|
|
expect(sha256("hello")).toMatch(/^[0-9a-f]{64}$/);
|
|
});
|
|
|
|
it("is deterministic for the same input", () => {
|
|
expect(sha256("test")).toBe(sha256("test"));
|
|
});
|
|
|
|
it("differs for different inputs", () => {
|
|
expect(sha256("a")).not.toBe(sha256("b"));
|
|
});
|
|
|
|
it("known vector: empty string", () => {
|
|
// SHA256("") = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
expect(sha256("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
|
});
|
|
});
|
|
|
|
// ─── Checkpoint ───────────────────────────────────────────────────────────────
|
|
|
|
describe("checkpoint round-trip", () => {
|
|
const TEST_MODULE = "__test_module_unit__";
|
|
const cpFile = resolve(process.cwd(), `.backfill-cursor-${TEST_MODULE}.json`);
|
|
|
|
afterEach(() => {
|
|
if (existsSync(cpFile)) unlinkSync(cpFile);
|
|
});
|
|
|
|
it("loadCheckpoint returns null when no file exists", () => {
|
|
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
|
});
|
|
|
|
it("saveCheckpoint + loadCheckpoint round-trip", () => {
|
|
const state = { cursor: "abc123", lastKey: "wordle:games:42" };
|
|
saveCheckpoint(TEST_MODULE, state);
|
|
expect(loadCheckpoint(TEST_MODULE)).toEqual(state);
|
|
});
|
|
|
|
it("clearCheckpoint removes the file", () => {
|
|
saveCheckpoint(TEST_MODULE, { cursor: "x", lastKey: "k" });
|
|
expect(existsSync(cpFile)).toBe(true);
|
|
clearCheckpoint(TEST_MODULE);
|
|
expect(existsSync(cpFile)).toBe(false);
|
|
});
|
|
|
|
it("clearCheckpoint is a no-op when file absent", () => {
|
|
expect(() => clearCheckpoint(TEST_MODULE)).not.toThrow();
|
|
});
|
|
|
|
it("loadCheckpoint returns null on corrupt JSON", () => {
|
|
writeFileSync(cpFile, "{bad json", "utf8");
|
|
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
|
});
|
|
});
|
|
|
|
// ─── countDiffRatio ───────────────────────────────────────────────────────────
|
|
|
|
describe("countDiffRatio", () => {
|
|
it("returns 0 for equal counts", () => {
|
|
expect(countDiffRatio(100, 100)).toBe(0);
|
|
});
|
|
|
|
it("returns 1 for (0, N)", () => {
|
|
expect(countDiffRatio(0, 100)).toBe(1);
|
|
});
|
|
|
|
it("returns 1 for (N, 0)", () => {
|
|
expect(countDiffRatio(50, 0)).toBe(1);
|
|
});
|
|
|
|
it("handles (0, 0) without division-by-zero — max(a,b,1)=1", () => {
|
|
expect(countDiffRatio(0, 0)).toBe(0);
|
|
});
|
|
|
|
it("1% diff on 1000", () => {
|
|
expect(countDiffRatio(1000, 990)).toBeCloseTo(0.01);
|
|
});
|
|
|
|
it("below 1% threshold for 5 diff on 1000", () => {
|
|
expect(countDiffRatio(1000, 995)).toBeLessThan(0.01);
|
|
});
|
|
});
|
|
|
|
// ─── sampleStrategy ───────────────────────────────────────────────────────────
|
|
|
|
describe("sampleStrategy", () => {
|
|
it("full-scan for 0 total", () => {
|
|
const { fullScan, sampleSize } = sampleStrategy(0);
|
|
expect(fullScan).toBe(true);
|
|
expect(sampleSize).toBe(0);
|
|
});
|
|
|
|
it("full-scan for 9999 total", () => {
|
|
const { fullScan, sampleSize } = sampleStrategy(9999);
|
|
expect(fullScan).toBe(true);
|
|
expect(sampleSize).toBe(9999);
|
|
});
|
|
|
|
it("full-scan boundary: 10000 triggers sampling", () => {
|
|
const { fullScan } = sampleStrategy(10000);
|
|
expect(fullScan).toBe(false);
|
|
});
|
|
|
|
it("sample size = ceil(sqrt(total)) for mid range", () => {
|
|
// sqrt(40000) = 200 → sampleSize = 200
|
|
const { sampleSize } = sampleStrategy(40000);
|
|
expect(sampleSize).toBe(200);
|
|
});
|
|
|
|
it("sample size capped at 500", () => {
|
|
// sqrt(1_000_000) = 1000, capped to 500
|
|
const { sampleSize } = sampleStrategy(1_000_000);
|
|
expect(sampleSize).toBe(500);
|
|
});
|
|
|
|
it("ceil applied: sqrt(10001) ≈ 100.005 → 101", () => {
|
|
const { sampleSize } = sampleStrategy(10201); // sqrt = 101 exactly
|
|
expect(sampleSize).toBe(101);
|
|
});
|
|
});
|
|
|
|
// ─── cfKvList ─────────────────────────────────────────────────────────────────
|
|
|
|
describe("cfKvList (fetch mock)", () => {
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
function mockFetch(body, status = 200) {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: status >= 200 && status < 300,
|
|
status,
|
|
statusText: status === 200 ? "OK" : "Error",
|
|
text: () => Promise.resolve(JSON.stringify(body)),
|
|
json: () => Promise.resolve(body),
|
|
}),
|
|
);
|
|
}
|
|
|
|
it("parses a successful list response", async () => {
|
|
mockFetch({
|
|
success: true,
|
|
result: [
|
|
{ name: "wordle:game:1", metadata: { expiration: 1700000000 } },
|
|
{ name: "wordle:game:2" },
|
|
],
|
|
result_info: { count: 2 },
|
|
});
|
|
const result = await cfKvList("acct", "ns", "tok", "wordle:", null);
|
|
expect(result.keys).toHaveLength(2);
|
|
expect(result.keys[0].name).toBe("wordle:game:1");
|
|
expect(result.keys[0].metadata?.expiration).toBe(1700000000);
|
|
expect(result.list_complete).toBe(true);
|
|
expect(result.cursor).toBeNull();
|
|
});
|
|
|
|
it("includes cursor when result_info.cursor present", async () => {
|
|
mockFetch({
|
|
success: true,
|
|
result: Array.from({ length: 1000 }, (_, i) => ({ name: `k:${i}` })),
|
|
result_info: { count: 1000, cursor: "next-page-cursor" },
|
|
});
|
|
const result = await cfKvList("acct", "ns", "tok", "k:", null);
|
|
expect(result.cursor).toBe("next-page-cursor");
|
|
expect(result.list_complete).toBe(false);
|
|
});
|
|
|
|
it("throws on HTTP error", async () => {
|
|
mockFetch({ errors: ["unauthorized"] }, 401);
|
|
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow("401");
|
|
});
|
|
|
|
it("throws when success=false", async () => {
|
|
mockFetch({ success: false, errors: [{ message: "namespace not found" }] });
|
|
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow(/error/i);
|
|
});
|
|
});
|
|
|
|
// ─── cfKvGet ──────────────────────────────────────────────────────────────────
|
|
|
|
describe("cfKvGet (fetch mock)", () => {
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
it("returns value text on success", async () => {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
text: () => Promise.resolve('{"score":42}'),
|
|
}),
|
|
);
|
|
const val = await cfKvGet("acct", "ns", "tok", "wordle:game:1");
|
|
expect(val).toBe('{"score":42}');
|
|
});
|
|
|
|
it("URL-encodes the key in the request URL", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
text: () => Promise.resolve("v"),
|
|
});
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvGet("acct", "ns", "tok", "some key/with spaces");
|
|
const calledUrl = fetchMock.mock.calls[0][0];
|
|
expect(calledUrl).toContain("some%20key%2Fwith%20spaces");
|
|
});
|
|
|
|
it("throws on HTTP 404", async () => {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 404,
|
|
statusText: "Not Found",
|
|
text: () => Promise.resolve("not found"),
|
|
}),
|
|
);
|
|
await expect(cfKvGet("acct", "ns", "tok", "missing:key")).rejects.toThrow("404");
|
|
});
|
|
});
|
|
|
|
// ─── getMongoClient teardown ──────────────────────────────────────────────────
|
|
|
|
// Ensure any singleton client opened during tests is closed (prevents open handles).
|
|
afterEach(async () => {
|
|
await closeMongoClient();
|
|
});
|