diff --git a/data/raw/10_LamDong_GNFT (1).xls.xlsx b/data/raw/10_LamDong_GNFT (1).xls.xlsx deleted file mode 100644 index 7c4467f..0000000 Binary files a/data/raw/10_LamDong_GNFT (1).xls.xlsx and /dev/null differ diff --git a/data/raw/update/2.BacKan_YQNX(1).xlsx b/data/raw/update/2.BacKan_YQNX(1).xlsx deleted file mode 100644 index a28e617..0000000 Binary files a/data/raw/update/2.BacKan_YQNX(1).xlsx and /dev/null differ diff --git a/scripts/check-duplicates.js b/scripts/check-duplicates.js new file mode 100644 index 0000000..1b5f70b --- /dev/null +++ b/scripts/check-duplicates.js @@ -0,0 +1,29 @@ +// One-off audit: detect content-identical Excel files via md5 +import crypto from "crypto"; +import fs from "fs"; +import path from "path"; + +const dirs = [ + "D:/tiennm99/thptqg2017/data/raw", + "D:/tiennm99/thptqg2017/data/raw/update", +]; + +const byHash = {}; +for (const d of dirs) { + for (const f of fs.readdirSync(d)) { + const full = path.join(d, f); + if (!fs.statSync(full).isFile() || !f.endsWith(".xlsx")) continue; + const h = crypto.createHash("md5").update(fs.readFileSync(full)).digest("hex"); + (byHash[h] ||= []).push(full); + } +} + +const total = Object.values(byHash).reduce((s, a) => s + a.length, 0); +const dupes = Object.entries(byHash).filter(([, a]) => a.length > 1); +console.log(`Total files: ${total}`); +console.log(`Unique by md5: ${Object.keys(byHash).length}`); +console.log(`Duplicate groups: ${dupes.length}`); +for (const [h, a] of dupes) { + console.log(` ${h.slice(0, 12)}:`); + a.forEach((p) => console.log(` ${p}`)); +}