mirror of
https://github.com/tiennm99/miti99bot-js.git
synced 2026-05-15 07:53:01 +00:00
3f03521e84
Pre-execution prerequisites for the Phase 07 cutover. Stage 2 of the cutover keeps DUAL_WRITE=0 for ~6 days; if anything regresses during that window the operator MUST be able to roll back to KV/D1 with the last N days of Mongo-only writes recovered. Pre-building these scripts (per code-reviewer #4) eliminates "draft a backfill under outage pressure" — the anti-pattern of writing untested code at 4am. Reverse-backfill - scripts/backfill-mongo-to-kv.js: full-scan Mongo collection per module, PUT each doc back to CF KV via REST. expiresAt → expirationTtl (clamped to 60s minimum per CF KV); already-expired docs are skipped (won't resurrect dead state). 50 ops/sec throttle. --dry-run + --module flags. - scripts/backfill-mongo-to-d1.js: full-scan trading_trades, build INSERT SQL preserving legacy_id where present (round-trips D1 autoincrement IDs preserved by phase-05 forward backfill). Sequential int generation for any docs without legacy_id. Pipes through wrangler d1 execute. - scripts/lib/migration-helpers.js: cfKvPut helper added. Delete guard (debugger #12) - scripts/wrangler-delete-guard.sh: interactive CONFIRM wrapper around wrangler kv namespace delete + wrangler d1 delete. Exits 3 when stdin is not a tty so it cannot run in CI. Documented: never run in CI. package.json: backfill:mongo:kv[:dry] + backfill:mongo:d1[:dry] scripts wired. Tests: 697 → 733 (+36). - 7 cfKvPut tests (REST URL, querystring, body, expiration_ttl param). - 10 reverse-KV TTL math tests (expired sentinel, future seconds, no-TTL, CF 60s minimum clamp). - 9 reverse-D1 SQL construction tests (escaping, legacy_id preservation, sequential generation). Lint clean. No Worker code touched. Stage 1 cutover, 7-day soak, snapshots, and Stage 3 cleanup (delete CFKVStore + simplify factories + edit package.json deploy chain) remain operator-driven and will be committed separately after binding deletion.
346 lines
12 KiB
JavaScript
346 lines
12 KiB
JavaScript
/**
|
|
* @file migration-helpers.test.js — pure-logic unit tests for migration-helpers.
|
|
*
|
|
* Tests: sha256, checkpoint round-trip, countDiffRatio, sampleStrategy,
|
|
* cfKvList / cfKvGet response parsing (via fetch mock).
|
|
*
|
|
* No real Atlas connection, no real CF REST calls.
|
|
*/
|
|
|
|
import { existsSync, unlinkSync, writeFileSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import {
|
|
cfKvGet,
|
|
cfKvList,
|
|
cfKvPut,
|
|
clearCheckpoint,
|
|
closeMongoClient,
|
|
countDiffRatio,
|
|
loadCheckpoint,
|
|
sampleStrategy,
|
|
saveCheckpoint,
|
|
sha256,
|
|
sleep,
|
|
} from "../../scripts/lib/migration-helpers.js";
|
|
|
|
// ─── sleep ────────────────────────────────────────────────────────────────────
|
|
|
|
describe("sleep", () => {
|
|
it("resolves after at least ms milliseconds", async () => {
|
|
const start = Date.now();
|
|
await sleep(20);
|
|
expect(Date.now() - start).toBeGreaterThanOrEqual(15); // allow 5ms jitter
|
|
});
|
|
});
|
|
|
|
// ─── sha256 ───────────────────────────────────────────────────────────────────
|
|
|
|
describe("sha256", () => {
|
|
it("produces a 64-char hex string", () => {
|
|
expect(sha256("hello")).toMatch(/^[0-9a-f]{64}$/);
|
|
});
|
|
|
|
it("is deterministic for the same input", () => {
|
|
expect(sha256("test")).toBe(sha256("test"));
|
|
});
|
|
|
|
it("differs for different inputs", () => {
|
|
expect(sha256("a")).not.toBe(sha256("b"));
|
|
});
|
|
|
|
it("known vector: empty string", () => {
|
|
// SHA256("") = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
expect(sha256("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
|
});
|
|
});
|
|
|
|
// ─── Checkpoint ───────────────────────────────────────────────────────────────
|
|
|
|
describe("checkpoint round-trip", () => {
|
|
const TEST_MODULE = "__test_module_unit__";
|
|
const cpFile = resolve(process.cwd(), `.backfill-cursor-${TEST_MODULE}.json`);
|
|
|
|
afterEach(() => {
|
|
if (existsSync(cpFile)) unlinkSync(cpFile);
|
|
});
|
|
|
|
it("loadCheckpoint returns null when no file exists", () => {
|
|
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
|
});
|
|
|
|
it("saveCheckpoint + loadCheckpoint round-trip", () => {
|
|
const state = { cursor: "abc123", lastKey: "wordle:games:42" };
|
|
saveCheckpoint(TEST_MODULE, state);
|
|
expect(loadCheckpoint(TEST_MODULE)).toEqual(state);
|
|
});
|
|
|
|
it("clearCheckpoint removes the file", () => {
|
|
saveCheckpoint(TEST_MODULE, { cursor: "x", lastKey: "k" });
|
|
expect(existsSync(cpFile)).toBe(true);
|
|
clearCheckpoint(TEST_MODULE);
|
|
expect(existsSync(cpFile)).toBe(false);
|
|
});
|
|
|
|
it("clearCheckpoint is a no-op when file absent", () => {
|
|
expect(() => clearCheckpoint(TEST_MODULE)).not.toThrow();
|
|
});
|
|
|
|
it("loadCheckpoint returns null on corrupt JSON", () => {
|
|
writeFileSync(cpFile, "{bad json", "utf8");
|
|
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
|
});
|
|
});
|
|
|
|
// ─── countDiffRatio ───────────────────────────────────────────────────────────
|
|
|
|
describe("countDiffRatio", () => {
|
|
it("returns 0 for equal counts", () => {
|
|
expect(countDiffRatio(100, 100)).toBe(0);
|
|
});
|
|
|
|
it("returns 1 for (0, N)", () => {
|
|
expect(countDiffRatio(0, 100)).toBe(1);
|
|
});
|
|
|
|
it("returns 1 for (N, 0)", () => {
|
|
expect(countDiffRatio(50, 0)).toBe(1);
|
|
});
|
|
|
|
it("handles (0, 0) without division-by-zero — max(a,b,1)=1", () => {
|
|
expect(countDiffRatio(0, 0)).toBe(0);
|
|
});
|
|
|
|
it("1% diff on 1000", () => {
|
|
expect(countDiffRatio(1000, 990)).toBeCloseTo(0.01);
|
|
});
|
|
|
|
it("below 1% threshold for 5 diff on 1000", () => {
|
|
expect(countDiffRatio(1000, 995)).toBeLessThan(0.01);
|
|
});
|
|
});
|
|
|
|
// ─── sampleStrategy ───────────────────────────────────────────────────────────
|
|
|
|
describe("sampleStrategy", () => {
|
|
it("full-scan for 0 total", () => {
|
|
const { fullScan, sampleSize } = sampleStrategy(0);
|
|
expect(fullScan).toBe(true);
|
|
expect(sampleSize).toBe(0);
|
|
});
|
|
|
|
it("full-scan for 9999 total", () => {
|
|
const { fullScan, sampleSize } = sampleStrategy(9999);
|
|
expect(fullScan).toBe(true);
|
|
expect(sampleSize).toBe(9999);
|
|
});
|
|
|
|
it("full-scan boundary: 10000 triggers sampling", () => {
|
|
const { fullScan } = sampleStrategy(10000);
|
|
expect(fullScan).toBe(false);
|
|
});
|
|
|
|
it("sample size = ceil(sqrt(total)) for mid range", () => {
|
|
// sqrt(40000) = 200 → sampleSize = 200
|
|
const { sampleSize } = sampleStrategy(40000);
|
|
expect(sampleSize).toBe(200);
|
|
});
|
|
|
|
it("sample size capped at 500", () => {
|
|
// sqrt(1_000_000) = 1000, capped to 500
|
|
const { sampleSize } = sampleStrategy(1_000_000);
|
|
expect(sampleSize).toBe(500);
|
|
});
|
|
|
|
it("ceil applied: sqrt(10001) ≈ 100.005 → 101", () => {
|
|
const { sampleSize } = sampleStrategy(10201); // sqrt = 101 exactly
|
|
expect(sampleSize).toBe(101);
|
|
});
|
|
});
|
|
|
|
// ─── cfKvList ─────────────────────────────────────────────────────────────────
|
|
|
|
describe("cfKvList (fetch mock)", () => {
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
function mockFetch(body, status = 200) {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: status >= 200 && status < 300,
|
|
status,
|
|
statusText: status === 200 ? "OK" : "Error",
|
|
text: () => Promise.resolve(JSON.stringify(body)),
|
|
json: () => Promise.resolve(body),
|
|
}),
|
|
);
|
|
}
|
|
|
|
it("parses a successful list response", async () => {
|
|
mockFetch({
|
|
success: true,
|
|
result: [
|
|
{ name: "wordle:game:1", metadata: { expiration: 1700000000 } },
|
|
{ name: "wordle:game:2" },
|
|
],
|
|
result_info: { count: 2 },
|
|
});
|
|
const result = await cfKvList("acct", "ns", "tok", "wordle:", null);
|
|
expect(result.keys).toHaveLength(2);
|
|
expect(result.keys[0].name).toBe("wordle:game:1");
|
|
expect(result.keys[0].metadata?.expiration).toBe(1700000000);
|
|
expect(result.list_complete).toBe(true);
|
|
expect(result.cursor).toBeNull();
|
|
});
|
|
|
|
it("includes cursor when result_info.cursor present", async () => {
|
|
mockFetch({
|
|
success: true,
|
|
result: Array.from({ length: 1000 }, (_, i) => ({ name: `k:${i}` })),
|
|
result_info: { count: 1000, cursor: "next-page-cursor" },
|
|
});
|
|
const result = await cfKvList("acct", "ns", "tok", "k:", null);
|
|
expect(result.cursor).toBe("next-page-cursor");
|
|
expect(result.list_complete).toBe(false);
|
|
});
|
|
|
|
it("throws on HTTP error", async () => {
|
|
mockFetch({ errors: ["unauthorized"] }, 401);
|
|
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow("401");
|
|
});
|
|
|
|
it("throws when success=false", async () => {
|
|
mockFetch({ success: false, errors: [{ message: "namespace not found" }] });
|
|
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow(/error/i);
|
|
});
|
|
});
|
|
|
|
// ─── cfKvGet ──────────────────────────────────────────────────────────────────
|
|
|
|
describe("cfKvGet (fetch mock)", () => {
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
it("returns value text on success", async () => {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
status: 200,
|
|
text: () => Promise.resolve('{"score":42}'),
|
|
}),
|
|
);
|
|
const val = await cfKvGet("acct", "ns", "tok", "wordle:game:1");
|
|
expect(val).toBe('{"score":42}');
|
|
});
|
|
|
|
it("URL-encodes the key in the request URL", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
text: () => Promise.resolve("v"),
|
|
});
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvGet("acct", "ns", "tok", "some key/with spaces");
|
|
const calledUrl = fetchMock.mock.calls[0][0];
|
|
expect(calledUrl).toContain("some%20key%2Fwith%20spaces");
|
|
});
|
|
|
|
it("throws on HTTP 404", async () => {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 404,
|
|
statusText: "Not Found",
|
|
text: () => Promise.resolve("not found"),
|
|
}),
|
|
);
|
|
await expect(cfKvGet("acct", "ns", "tok", "missing:key")).rejects.toThrow("404");
|
|
});
|
|
});
|
|
|
|
// ─── cfKvPut ──────────────────────────────────────────────────────────────────
|
|
|
|
describe("cfKvPut (fetch mock)", () => {
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
function mockFetch(status = 200) {
|
|
vi.stubGlobal(
|
|
"fetch",
|
|
vi.fn().mockResolvedValue({
|
|
ok: status >= 200 && status < 300,
|
|
status,
|
|
text: () => Promise.resolve(""),
|
|
}),
|
|
);
|
|
}
|
|
|
|
it("issues a PUT request to the correct URL", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "my:key", "myvalue");
|
|
const [url, init] = fetchMock.mock.calls[0];
|
|
expect(url).toContain("/accounts/acct/storage/kv/namespaces/ns/values/my%3Akey");
|
|
expect(init.method).toBe("PUT");
|
|
});
|
|
|
|
it("sends Authorization Bearer header", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "TOKEN", "k", "v");
|
|
const init = fetchMock.mock.calls[0][1];
|
|
expect(init.headers.Authorization).toBe("Bearer TOKEN");
|
|
});
|
|
|
|
it("sends the value as the request body", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "k", "hello-value");
|
|
const init = fetchMock.mock.calls[0][1];
|
|
expect(init.body).toBe("hello-value");
|
|
});
|
|
|
|
it("appends expiration_ttl query param when provided", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "k", "v", { expirationTtl: 300 });
|
|
const url = fetchMock.mock.calls[0][0];
|
|
expect(url).toContain("expiration_ttl=300");
|
|
});
|
|
|
|
it("omits expiration_ttl query param when opts is empty", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "k", "v", {});
|
|
const url = fetchMock.mock.calls[0][0];
|
|
expect(url).not.toContain("expiration_ttl");
|
|
});
|
|
|
|
it("omits expiration_ttl when opts is omitted entirely", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "k", "v");
|
|
const url = fetchMock.mock.calls[0][0];
|
|
expect(url).not.toContain("expiration_ttl");
|
|
});
|
|
|
|
it("throws on HTTP error response", async () => {
|
|
mockFetch(403);
|
|
await expect(cfKvPut("acct", "ns", "tok", "k", "v")).rejects.toThrow("403");
|
|
});
|
|
|
|
it("URL-encodes special characters in key", async () => {
|
|
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
|
vi.stubGlobal("fetch", fetchMock);
|
|
await cfKvPut("acct", "ns", "tok", "key with spaces/slash", "v");
|
|
const url = fetchMock.mock.calls[0][0];
|
|
expect(url).toContain("key%20with%20spaces%2Fslash");
|
|
});
|
|
});
|
|
|
|
// ─── getMongoClient teardown ──────────────────────────────────────────────────
|
|
|
|
// Ensure any singleton client opened during tests is closed (prevents open handles).
|
|
afterEach(async () => {
|
|
await closeMongoClient();
|
|
});
|