mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-08-14 01:24:20 +00:00
refactor: collapse the two projects into one tree and move to npm
The repo held two near-duplicate projects. 2016/ and 2017/ each carried their
own React frontend, their own copy of the same Rust crate, and their own
package manager setup. 2016/tools/sync-from-thptqg2017.sh existed purely to
copy the parser source between them.
New layout:
index.html + src/ the 2017 frontend, now the only one
data/<id>/ 2016, 2017, 2017-old, 2017-old2
parser/ the single Rust crate, configs renamed to <id>.toml
docs/ both projects' docs, 2016 copies suffixed -2016-legacy
pending the merge pass
<id> is now one identifier end to end: data/<id>/ feeds parser/configs/<id>.toml
and produces db/<id>.db.gz.
pnpm gives way to npm. pnpm-workspace.yaml existed only to whitelist
better-sqlite3's native build, which npm permits by default, so it has no
equivalent and is simply gone. Lockfiles cannot be converted; package-lock.json
is generated fresh. The migration direction is safe — pnpm's strict layout
forbids phantom dependencies, so anything that resolved under pnpm resolves
under npm's flat tree.
Adds parser/scripts/build-db.js and src/datasets.js: the four dataset IDs are
declared once and read by both the build tooling and (from the next phase) the
frontend.
Follow-on fixes the move made necessary:
- eslint's Node-globals override pointed at scripts/, now parser/scripts/
- crawl-baotintuc.js wrote to <root>/data, now data/2017
- golden tests loaded configs by their old thptqg*-data.toml names
Drops the #[ignore]d Rust-vs-Node golden test. It shelled out to pnpm to run
scripts/build-database.js, a file removed when the parser was ported to Rust,
so it could never pass. check-duplicates.js and diff-datasets.js were already
broken before this change and are annotated as such rather than half-fixed.
63 Rust tests pass and clippy is clean from the new location.
This commit is contained in:
+21
@@ -0,0 +1,21 @@
|
||||
### Editors ###
|
||||
.idea
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
### Node ###
|
||||
node_modules/
|
||||
|
||||
### Build output ###
|
||||
dist/
|
||||
|
||||
### Generated databases and Vite publicDir staging ###
|
||||
.build/
|
||||
|
||||
### Rust build artefacts ###
|
||||
parser/target/
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
.stylelintcache
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Sveltekit cache directory
|
||||
.svelte-kit/
|
||||
|
||||
# vitepress build output
|
||||
**/.vitepress/dist
|
||||
|
||||
# vitepress cache directory
|
||||
**/.vitepress/cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# Firebase cache directory
|
||||
.firebase/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
.vscode-test
|
||||
|
||||
# yarn v3
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
|
||||
# Vite logs files
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
|
||||
# Generated database files
|
||||
public/*.db
|
||||
public/*.db.gz
|
||||
|
||||
# Rust build artifacts
|
||||
tools/xlsxread/target/
|
||||
@@ -1,57 +0,0 @@
|
||||
# thptqg2016
|
||||
|
||||
Lookup tool for Vietnam's 2016 National High School Graduation Exam (THPT Quốc gia) scores — 877,461 candidates nationwide.
|
||||
|
||||
Fully static app running entirely in the browser (SQLite via `sql.js`). No backend, no query logging.
|
||||
|
||||
## Features
|
||||
|
||||
- **Quick lookup** by exam ID (`số báo danh`) or full name (diacritics-insensitive)
|
||||
- **Custom read-only SQL** (SELECT / PRAGMA / EXPLAIN / WITH) with 7 built-in preset queries
|
||||
- **Complete data**: 12 subjects (Math, Literature, Physics, Chemistry, Biology, History, Geography, English, French, German, Japanese, Chinese), exam cluster, date of birth, gender
|
||||
- Safety caps: 100 rows for lookup, 1000 rows for custom SQL
|
||||
- Dark mode, `Ctrl+Enter` shortcut to run queries
|
||||
|
||||
## Demo
|
||||
|
||||
<https://tiennm99.github.io/thptqg/2016/>
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Build the SQLite database from source Excel files (requires Rust stable)
|
||||
pnpm run build:db
|
||||
|
||||
# Or build in two steps:
|
||||
pnpm run build:rust # compile the xlsxread binary
|
||||
./tools/xlsxread/target/release/xlsxread build \
|
||||
--schema tools/xlsxread/configs/thptqg2016-data.toml \
|
||||
--input data \
|
||||
--output public/thptqg2016.db
|
||||
|
||||
pnpm run dev # Vite dev server
|
||||
pnpm run build # Production bundle → dist/
|
||||
pnpm run lint # ESLint
|
||||
```
|
||||
|
||||
The database is built by the `xlsxread` Rust binary (`tools/xlsxread/`), which
|
||||
reads the 119 mixed `.xls`/`.xlsx` files and auto-detects the column layout per
|
||||
file (`separate-scores`, `mapped`, or positional default). No Node.js Excel
|
||||
library is required at build time.
|
||||
|
||||
The GitHub Actions workflow (`.github/workflows/deploy.yml`) compiles
|
||||
`xlsxread`, builds the DB, gzips it, and deploys to GitHub Pages on every push
|
||||
to `main`.
|
||||
|
||||
## Tech stack
|
||||
|
||||
React 19 · Vite · sql.js (WASM) · xlsxread (Rust, build-time) · GitHub Pages
|
||||
|
||||
## Documentation
|
||||
|
||||
- [Project overview (PDR)](./docs/project-overview-pdr.md)
|
||||
- [Codebase summary](./docs/codebase-summary.md)
|
||||
- [System architecture](./docs/system-architecture.md)
|
||||
- [Deployment guide](./docs/deployment-guide.md)
|
||||
|
||||
**Source**: Originally published at <https://dtntbacgiang.edu.vn/tin-tuc/tin-tuc-su-kien/cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html> (currently inaccessible). Data is for reference only.
|
||||
@@ -1,31 +0,0 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import reactHooks from "eslint-plugin-react-hooks";
|
||||
import reactRefresh from "eslint-plugin-react-refresh";
|
||||
|
||||
export default [
|
||||
{ ignores: ["dist"] },
|
||||
{
|
||||
files: ["**/*.{js,jsx}"],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2024,
|
||||
globals: globals.browser,
|
||||
parserOptions: {
|
||||
ecmaVersion: "latest",
|
||||
ecmaFeatures: { jsx: true },
|
||||
sourceType: "module",
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
"react-hooks": reactHooks,
|
||||
"react-refresh": reactRefresh,
|
||||
},
|
||||
rules: {
|
||||
...reactHooks.configs.recommended.rules,
|
||||
"react-refresh/only-export-components": [
|
||||
"warn",
|
||||
{ allowConstantExport: true },
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -1,13 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Tra cứu điểm thi THPT QG 2016</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
Generated
-1624
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
allowBuilds:
|
||||
better-sqlite3: true
|
||||
@@ -1,474 +0,0 @@
|
||||
.app {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem 1rem 3rem;
|
||||
}
|
||||
|
||||
/* Header */
|
||||
header {
|
||||
text-align: center;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
font-size: clamp(1.4rem, 2.4vw, 2rem);
|
||||
color: var(--text);
|
||||
margin: 0 0 0.4rem;
|
||||
letter-spacing: -0.01em;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.95rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.subtitle .pill {
|
||||
display: inline-block;
|
||||
padding: 0.1rem 0.55rem;
|
||||
background: var(--accent-soft);
|
||||
color: var(--accent);
|
||||
border-radius: 999px;
|
||||
font-weight: 600;
|
||||
font-size: 0.8rem;
|
||||
margin-right: 0.35rem;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: 0;
|
||||
max-width: 600px;
|
||||
margin: 0 auto 1.5rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.tab {
|
||||
flex: 1;
|
||||
padding: 0.75rem 1rem;
|
||||
font-size: 0.95rem;
|
||||
background: none;
|
||||
border: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
margin-bottom: -1px;
|
||||
cursor: pointer;
|
||||
color: var(--text-muted);
|
||||
font-weight: 500;
|
||||
transition: color 0.15s, border-color 0.15s;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.tab[aria-selected="true"] {
|
||||
color: var(--accent);
|
||||
border-bottom-color: var(--accent);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Search form */
|
||||
.search-form {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
max-width: 600px;
|
||||
margin: 0 auto 1.5rem;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.search-field {
|
||||
flex: 1;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.search-form input {
|
||||
width: 100%;
|
||||
padding: 0.75rem 2.25rem 0.75rem 1rem;
|
||||
font-size: 1rem;
|
||||
border: 1px solid var(--border-strong);
|
||||
border-radius: var(--radius);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
outline: none;
|
||||
transition: border-color 0.15s, box-shadow 0.15s;
|
||||
}
|
||||
|
||||
.search-form input:focus {
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 3px var(--accent-soft);
|
||||
}
|
||||
|
||||
.search-clear {
|
||||
position: absolute;
|
||||
right: 0.4rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
width: 1.75rem;
|
||||
height: 1.75rem;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--text-faint);
|
||||
font-size: 1.1rem;
|
||||
border-radius: 50%;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.search-clear:hover {
|
||||
background: var(--surface-alt);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.search-form button[type="submit"] {
|
||||
padding: 0.75rem 1.5rem;
|
||||
font-size: 0.95rem;
|
||||
font-weight: 600;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, transform 0.05s;
|
||||
}
|
||||
|
||||
.search-form button[type="submit"]:hover:not(:disabled) {
|
||||
background: var(--accent-hover);
|
||||
}
|
||||
|
||||
.search-form button[type="submit"]:active:not(:disabled) {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.search-form button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* Custom query */
|
||||
.custom-query {
|
||||
max-width: 960px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.preset-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.4rem;
|
||||
margin-bottom: 0.85rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.preset-label {
|
||||
font-size: 0.8rem;
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
margin-right: 0.25rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.preset-btn {
|
||||
padding: 0.35rem 0.75rem;
|
||||
font-size: 0.8rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: 999px;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, border-color 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.preset-btn:hover:not(:disabled) {
|
||||
background: var(--accent-soft);
|
||||
border-color: var(--accent);
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.preset-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.query-form {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.query-form textarea {
|
||||
width: 100%;
|
||||
padding: 0.85rem 1rem;
|
||||
font-size: 0.9rem;
|
||||
font-family:
|
||||
"JetBrains Mono", "Cascadia Code", "Fira Code", ui-monospace, monospace;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--border-strong);
|
||||
border-radius: var(--radius);
|
||||
outline: none;
|
||||
resize: vertical;
|
||||
min-height: 130px;
|
||||
transition: border-color 0.15s, box-shadow 0.15s;
|
||||
tab-size: 2;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.query-form textarea:focus {
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 3px var(--accent-soft);
|
||||
}
|
||||
|
||||
.query-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
margin-top: 0.6rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.query-actions button {
|
||||
padding: 0.6rem 1.4rem;
|
||||
font-size: 0.9rem;
|
||||
font-weight: 600;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.query-actions button:hover:not(:disabled) {
|
||||
background: var(--accent-hover);
|
||||
}
|
||||
|
||||
.query-actions button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.exec-time {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.exec-time.running {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.kbd {
|
||||
display: inline-block;
|
||||
padding: 0.05rem 0.4rem;
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.75rem;
|
||||
background: var(--surface-alt);
|
||||
border: 1px solid var(--border);
|
||||
border-bottom-width: 2px;
|
||||
border-radius: 4px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
/* Loading */
|
||||
.loading {
|
||||
max-width: 600px;
|
||||
margin: 0.5rem auto 1.5rem;
|
||||
padding: 0.85rem 1rem;
|
||||
text-align: center;
|
||||
color: var(--text-muted);
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.loading p {
|
||||
margin: 0 0 0.5rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
width: 100%;
|
||||
max-width: 300px;
|
||||
height: 6px;
|
||||
background: var(--surface-alt);
|
||||
border-radius: 4px;
|
||||
margin: 0 auto;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, var(--accent), var(--accent-hover));
|
||||
border-radius: 4px;
|
||||
transition: width 0.3s;
|
||||
}
|
||||
|
||||
/* Results table */
|
||||
.table-wrapper {
|
||||
overflow-x: auto;
|
||||
margin-top: 1rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.result-count {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.9rem;
|
||||
margin: 0.5rem 0;
|
||||
padding: 0 0.25rem;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
thead th {
|
||||
background: var(--surface-alt);
|
||||
padding: 0.65rem 0.6rem;
|
||||
text-align: left;
|
||||
white-space: nowrap;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
border-bottom: 1px solid var(--border-strong);
|
||||
color: var(--text-muted);
|
||||
font-weight: 600;
|
||||
font-size: 0.8rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
tbody td {
|
||||
padding: 0.55rem 0.6rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
tbody tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
tbody tr:nth-child(even) td {
|
||||
background: color-mix(in srgb, var(--surface-alt) 45%, transparent);
|
||||
}
|
||||
|
||||
tbody tr:hover td {
|
||||
background: var(--accent-soft);
|
||||
}
|
||||
|
||||
.sbd-cell {
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.name-cell {
|
||||
white-space: nowrap;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.cumthi-cell {
|
||||
font-size: 0.82rem;
|
||||
color: var(--text-muted);
|
||||
max-width: 220px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.score-cell {
|
||||
text-align: center;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.score-cell.score-high {
|
||||
color: var(--ok);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.score-cell.score-low {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.score-cell.score-empty {
|
||||
color: var(--text-faint);
|
||||
}
|
||||
|
||||
/* Messages */
|
||||
.no-results,
|
||||
.error,
|
||||
.warning {
|
||||
text-align: center;
|
||||
padding: 0.9rem 1rem;
|
||||
border-radius: var(--radius);
|
||||
margin: 1rem auto;
|
||||
max-width: 600px;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.no-results {
|
||||
color: var(--text-muted);
|
||||
background: var(--surface);
|
||||
border: 1px dashed var(--border-strong);
|
||||
}
|
||||
|
||||
.error {
|
||||
color: var(--danger);
|
||||
background: var(--danger-bg);
|
||||
border: 1px solid color-mix(in srgb, var(--danger) 25%, transparent);
|
||||
}
|
||||
|
||||
.warning {
|
||||
color: var(--warn);
|
||||
background: var(--warn-bg);
|
||||
}
|
||||
|
||||
/* Footer */
|
||||
footer {
|
||||
text-align: center;
|
||||
margin-top: 3rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid var(--border);
|
||||
color: var(--text-faint);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
footer a {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 640px) {
|
||||
.app {
|
||||
padding: 1.25rem 0.75rem 2rem;
|
||||
}
|
||||
|
||||
.search-form {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.search-form button[type="submit"] {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
table {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
thead th,
|
||||
tbody td {
|
||||
padding: 0.45rem 0.4rem;
|
||||
}
|
||||
|
||||
.preset-list {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.preset-btn {
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
import { useState, useCallback, useEffect } from "react";
|
||||
import { useSqlite } from "./hooks/use-sqlite";
|
||||
import { SearchForm } from "./components/search-form";
|
||||
import { ScoreTable } from "./components/score-table";
|
||||
import { CustomQuery } from "./components/custom-query";
|
||||
import "./App.css";
|
||||
|
||||
const DB_URL = import.meta.env.BASE_URL + "thptqg2016.db.gz";
|
||||
const MAX_RESULTS = 100;
|
||||
|
||||
// Strip Vietnamese diacritics for search: "nguyễn bữu lộc" → "nguyen buu loc"
|
||||
function toAscii(str) {
|
||||
return str
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/đ/gi, "d")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// Check if string contains only ASCII (no Vietnamese diacritics)
|
||||
function isAsciiOnly(str) {
|
||||
return /^[\x00-\x7F]*$/.test(str);
|
||||
}
|
||||
|
||||
function App() {
|
||||
const { db, loading, error, progress } = useSqlite(DB_URL);
|
||||
const [results, setResults] = useState(null);
|
||||
const [searchError, setSearchError] = useState(null);
|
||||
const [activeTab, setActiveTab] = useState("search");
|
||||
|
||||
const handleSearch = useCallback(
|
||||
(query) => {
|
||||
if (!db) return;
|
||||
setSearchError(null);
|
||||
|
||||
try {
|
||||
const isExamId = /^[A-Z]{2,4}\d+$/i.test(query) || /^\d+$/.test(query);
|
||||
let stmt;
|
||||
|
||||
if (isExamId) {
|
||||
stmt = db.prepare(
|
||||
"SELECT * FROM student WHERE so_bao_danh = $q LIMIT $limit",
|
||||
);
|
||||
stmt.bind({ $q: query.toUpperCase(), $limit: MAX_RESULTS });
|
||||
} else if (isAsciiOnly(query)) {
|
||||
// ASCII input: search against normalized column (diacritics-insensitive)
|
||||
stmt = db.prepare(
|
||||
"SELECT * FROM student WHERE ho_ten_ascii LIKE $q LIMIT $limit",
|
||||
);
|
||||
stmt.bind({ $q: `%${toAscii(query)}%`, $limit: MAX_RESULTS });
|
||||
} else {
|
||||
// Vietnamese input: search both original and normalized
|
||||
const normalized = toAscii(query);
|
||||
stmt = db.prepare(
|
||||
`SELECT * FROM student
|
||||
WHERE ho_ten LIKE $q OR ho_ten_ascii LIKE $qn
|
||||
LIMIT $limit`,
|
||||
);
|
||||
stmt.bind({
|
||||
$q: `%${query}%`,
|
||||
$qn: `%${normalized}%`,
|
||||
$limit: MAX_RESULTS,
|
||||
});
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
setResults(rows);
|
||||
} catch (err) {
|
||||
setSearchError(err.message);
|
||||
}
|
||||
},
|
||||
[db],
|
||||
);
|
||||
|
||||
// Ctrl+Enter shortcut to execute query in SQL tab
|
||||
useEffect(() => {
|
||||
function handleKeyDown(e) {
|
||||
if (e.ctrlKey && e.key === "Enter" && activeTab === "sql") {
|
||||
const form = document.querySelector(".query-form");
|
||||
if (form) form.requestSubmit();
|
||||
}
|
||||
}
|
||||
window.addEventListener("keydown", handleKeyDown);
|
||||
return () => window.removeEventListener("keydown", handleKeyDown);
|
||||
}, [activeTab]);
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header>
|
||||
<h1>Tra cứu điểm thi THPT Quốc gia 2016</h1>
|
||||
<p className="subtitle">
|
||||
<span className="pill">877.461 thí sinh</span>
|
||||
Dữ liệu toàn quốc · Hỗ trợ truy vấn SQL tùy chỉnh
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
{loading && (
|
||||
<div className="loading" role="status" aria-live="polite">
|
||||
<p>
|
||||
Đang tải cơ sở dữ liệu{progress > 0 ? ` · ${progress}%` : "..."}
|
||||
</p>
|
||||
<div className="progress-bar">
|
||||
<div
|
||||
className="progress-fill"
|
||||
style={{ width: `${progress}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && <p className="error">Lỗi: {error}</p>}
|
||||
|
||||
<div className="tabs" role="tablist">
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={activeTab === "search"}
|
||||
className="tab"
|
||||
onClick={() => setActiveTab("search")}
|
||||
>
|
||||
Tra cứu
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={activeTab === "sql"}
|
||||
className="tab"
|
||||
onClick={() => setActiveTab("sql")}
|
||||
>
|
||||
Truy vấn SQL
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{activeTab === "search" && (
|
||||
<>
|
||||
<SearchForm
|
||||
onSearch={handleSearch}
|
||||
disabled={loading || !!error}
|
||||
/>
|
||||
|
||||
{searchError && (
|
||||
<p className="error">Lỗi truy vấn: {searchError}</p>
|
||||
)}
|
||||
|
||||
<ScoreTable results={results} />
|
||||
|
||||
{results && results.length >= MAX_RESULTS && (
|
||||
<p className="warning">
|
||||
Hiển thị tối đa {MAX_RESULTS} kết quả. Vui lòng tìm kiếm cụ
|
||||
thể hơn.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{activeTab === "sql" && (
|
||||
<CustomQuery db={db} disabled={loading || !!error} />
|
||||
)}
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>
|
||||
Nguồn: Sưu tầm từ trang báo thời đó · Dữ liệu chỉ mang tính tham
|
||||
khảo
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -1,219 +0,0 @@
|
||||
import { useState, useCallback } from "react";
|
||||
|
||||
const MAX_ROWS = 1000;
|
||||
|
||||
const PRESET_QUERIES = [
|
||||
{
|
||||
label: "Top 10 điểm Toán cao nhất",
|
||||
sql: `SELECT so_bao_danh, ho_ten, ten_cum_thi, toan
|
||||
FROM student WHERE toan IS NOT NULL
|
||||
ORDER BY toan DESC LIMIT 10`,
|
||||
},
|
||||
{
|
||||
label: "Điểm trung bình theo cụm thi",
|
||||
sql: `SELECT ten_cum_thi,
|
||||
COUNT(*) AS so_luong,
|
||||
ROUND(AVG(toan), 2) AS tb_toan,
|
||||
ROUND(AVG(ngu_van), 2) AS tb_van,
|
||||
ROUND(AVG(tieng_anh), 2) AS tb_anh
|
||||
FROM student
|
||||
GROUP BY ten_cum_thi
|
||||
ORDER BY so_luong DESC
|
||||
LIMIT 20`,
|
||||
},
|
||||
{
|
||||
label: "Thống kê theo giới tính",
|
||||
sql: `SELECT gioi_tinh,
|
||||
COUNT(*) AS so_luong,
|
||||
ROUND(AVG(toan), 2) AS tb_toan,
|
||||
ROUND(AVG(ngu_van), 2) AS tb_van
|
||||
FROM student
|
||||
WHERE gioi_tinh IS NOT NULL
|
||||
GROUP BY gioi_tinh`,
|
||||
},
|
||||
{
|
||||
label: "Thí sinh đạt 9+ điểm Toán",
|
||||
sql: `SELECT so_bao_danh, ho_ten, ten_cum_thi, gioi_tinh, toan
|
||||
FROM student WHERE toan >= 9
|
||||
ORDER BY toan DESC LIMIT 50`,
|
||||
},
|
||||
{
|
||||
label: "Phân bố điểm Toán",
|
||||
sql: `SELECT
|
||||
CASE
|
||||
WHEN toan < 1 THEN '0-1'
|
||||
WHEN toan < 2 THEN '1-2'
|
||||
WHEN toan < 3 THEN '2-3'
|
||||
WHEN toan < 4 THEN '3-4'
|
||||
WHEN toan < 5 THEN '4-5'
|
||||
WHEN toan < 6 THEN '5-6'
|
||||
WHEN toan < 7 THEN '6-7'
|
||||
WHEN toan < 8 THEN '7-8'
|
||||
WHEN toan < 9 THEN '8-9'
|
||||
ELSE '9-10'
|
||||
END AS khoang_diem,
|
||||
COUNT(*) AS so_luong
|
||||
FROM student WHERE toan IS NOT NULL
|
||||
GROUP BY khoang_diem
|
||||
ORDER BY khoang_diem`,
|
||||
},
|
||||
{
|
||||
label: "Số thí sinh theo ngoại ngữ",
|
||||
sql: `SELECT
|
||||
SUM(CASE WHEN tieng_anh IS NOT NULL THEN 1 ELSE 0 END) AS tieng_anh,
|
||||
SUM(CASE WHEN tieng_phap IS NOT NULL THEN 1 ELSE 0 END) AS tieng_phap,
|
||||
SUM(CASE WHEN tieng_duc IS NOT NULL THEN 1 ELSE 0 END) AS tieng_duc,
|
||||
SUM(CASE WHEN tieng_nhat IS NOT NULL THEN 1 ELSE 0 END) AS tieng_nhat,
|
||||
SUM(CASE WHEN tieng_trung IS NOT NULL THEN 1 ELSE 0 END) AS tieng_trung
|
||||
FROM student`,
|
||||
},
|
||||
{
|
||||
label: "Schema bảng student",
|
||||
sql: `PRAGMA table_info(student)`,
|
||||
},
|
||||
];
|
||||
|
||||
export function CustomQuery({ db, disabled }) {
|
||||
const [sql, setSql] = useState("");
|
||||
const [columns, setColumns] = useState([]);
|
||||
const [rows, setRows] = useState([]);
|
||||
const [queryError, setQueryError] = useState(null);
|
||||
const [execTime, setExecTime] = useState(null);
|
||||
|
||||
const executeQuery = useCallback(
|
||||
(queryStr) => {
|
||||
if (!db) return;
|
||||
setQueryError(null);
|
||||
setColumns([]);
|
||||
setRows([]);
|
||||
setExecTime(null);
|
||||
|
||||
const trimmed = queryStr.trim();
|
||||
if (!trimmed) return;
|
||||
|
||||
// Safety: only allow read-only statements
|
||||
const upper = trimmed.toUpperCase();
|
||||
const allowed = ["SELECT", "PRAGMA", "EXPLAIN", "WITH"];
|
||||
if (!allowed.some((kw) => upper.startsWith(kw))) {
|
||||
setQueryError(
|
||||
"Chỉ hỗ trợ truy vấn đọc (SELECT, PRAGMA, EXPLAIN, WITH).",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Auto-add LIMIT if user forgot
|
||||
let finalSql = trimmed;
|
||||
if (
|
||||
upper.startsWith("SELECT") &&
|
||||
!upper.includes("LIMIT") &&
|
||||
!upper.includes("PRAGMA")
|
||||
) {
|
||||
finalSql = `${trimmed.replace(/;$/, "")} LIMIT ${MAX_ROWS}`;
|
||||
}
|
||||
|
||||
try {
|
||||
const start = performance.now();
|
||||
const stmt = db.prepare(finalSql);
|
||||
const colNames = stmt.getColumnNames();
|
||||
const resultRows = [];
|
||||
|
||||
let count = 0;
|
||||
while (stmt.step() && count < MAX_ROWS) {
|
||||
resultRows.push(stmt.get());
|
||||
count++;
|
||||
}
|
||||
stmt.free();
|
||||
|
||||
const elapsed = performance.now() - start;
|
||||
setColumns(colNames);
|
||||
setRows(resultRows);
|
||||
setExecTime(elapsed.toFixed(1));
|
||||
} catch (err) {
|
||||
setQueryError(err.message);
|
||||
}
|
||||
},
|
||||
[db],
|
||||
);
|
||||
|
||||
function handleSubmit(e) {
|
||||
e.preventDefault();
|
||||
executeQuery(sql);
|
||||
}
|
||||
|
||||
function handlePreset(presetSql) {
|
||||
setSql(presetSql);
|
||||
executeQuery(presetSql);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="custom-query">
|
||||
<div className="preset-list">
|
||||
<span className="preset-label">Mẫu truy vấn:</span>
|
||||
{PRESET_QUERIES.map((p, i) => (
|
||||
<button
|
||||
key={i}
|
||||
className="preset-btn"
|
||||
onClick={() => handlePreset(p.sql)}
|
||||
disabled={disabled}
|
||||
>
|
||||
{p.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<form onSubmit={handleSubmit} className="query-form">
|
||||
<textarea
|
||||
value={sql}
|
||||
onChange={(e) => setSql(e.target.value)}
|
||||
placeholder={`Nhập truy vấn SQL...\nVí dụ: SELECT * FROM student WHERE toan >= 9 LIMIT 10`}
|
||||
disabled={disabled}
|
||||
rows={5}
|
||||
spellCheck={false}
|
||||
/>
|
||||
<div className="query-actions">
|
||||
<button type="submit" disabled={disabled || !sql.trim()}>
|
||||
Thực thi <span className="kbd">Ctrl</span>+
|
||||
<span className="kbd">Enter</span>
|
||||
</button>
|
||||
{execTime !== null && (
|
||||
<span className="exec-time">
|
||||
{rows.length} kết quả · {execTime}ms
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{queryError && <p className="error">Lỗi: {queryError}</p>}
|
||||
|
||||
{columns.length > 0 && (
|
||||
<div className="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
{columns.map((col, i) => (
|
||||
<th key={i}>{col}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((row, ri) => (
|
||||
<tr key={ri}>
|
||||
{row.map((cell, ci) => (
|
||||
<td key={ci} className="score-cell">
|
||||
{cell === null ? "NULL" : String(cell)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{rows.length >= MAX_ROWS && (
|
||||
<p className="warning">
|
||||
Hiển thị tối đa {MAX_ROWS} kết quả. Thêm LIMIT để giới hạn.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
const SUBJECT_COLUMNS = [
|
||||
{ key: "toan", label: "Toán" },
|
||||
{ key: "ngu_van", label: "Ngữ văn" },
|
||||
{ key: "vat_ly", label: "Vật lí" },
|
||||
{ key: "hoa_hoc", label: "Hóa học" },
|
||||
{ key: "sinh_hoc", label: "Sinh học" },
|
||||
{ key: "lich_su", label: "Lịch sử" },
|
||||
{ key: "dia_ly", label: "Địa lí" },
|
||||
{ key: "tieng_anh", label: "T.Anh" },
|
||||
{ key: "tieng_phap", label: "T.Pháp" },
|
||||
{ key: "tieng_duc", label: "T.Đức" },
|
||||
{ key: "tieng_nhat", label: "T.Nhật" },
|
||||
{ key: "tieng_trung", label: "T.Trung" },
|
||||
];
|
||||
|
||||
function formatScore(val) {
|
||||
if (val === null || val === undefined) return "—";
|
||||
return Number(val).toFixed(2);
|
||||
}
|
||||
|
||||
function scoreClass(val) {
|
||||
if (val === null || val === undefined) return "score-cell score-empty";
|
||||
const num = Number(val);
|
||||
if (num >= 8) return "score-cell score-high";
|
||||
if (num < 4) return "score-cell score-low";
|
||||
return "score-cell";
|
||||
}
|
||||
|
||||
export function ScoreTable({ results }) {
|
||||
if (!results) return null;
|
||||
if (results.length === 0) {
|
||||
return <p className="no-results">Không tìm thấy kết quả.</p>;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<p className="result-count">Tìm thấy {results.length} kết quả</p>
|
||||
<div className="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>SBD</th>
|
||||
<th>Họ tên</th>
|
||||
<th>Ngày sinh</th>
|
||||
<th>Cụm thi</th>
|
||||
<th>GT</th>
|
||||
{SUBJECT_COLUMNS.map((col) => (
|
||||
<th key={col.key}>{col.label}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{results.map((row) => (
|
||||
<tr key={row.so_bao_danh}>
|
||||
<td className="sbd-cell">{row.so_bao_danh}</td>
|
||||
<td className="name-cell">{row.ho_ten}</td>
|
||||
<td>{row.ngay_sinh || "—"}</td>
|
||||
<td className="cumthi-cell" title={row.ten_cum_thi || ""}>
|
||||
{row.ten_cum_thi || "—"}
|
||||
</td>
|
||||
<td>{row.gioi_tinh || "—"}</td>
|
||||
{SUBJECT_COLUMNS.map((col) => (
|
||||
<td key={col.key} className={scoreClass(row[col.key])}>
|
||||
{formatScore(row[col.key])}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import { useState } from "react";
|
||||
|
||||
export function SearchForm({ onSearch, disabled }) {
|
||||
const [query, setQuery] = useState("");
|
||||
|
||||
function handleSubmit(e) {
|
||||
e.preventDefault();
|
||||
const trimmed = query.trim();
|
||||
if (trimmed) onSearch(trimmed);
|
||||
}
|
||||
|
||||
return (
|
||||
<form onSubmit={handleSubmit} className="search-form" role="search">
|
||||
<div className="search-field">
|
||||
<label htmlFor="search-input" className="visually-hidden" hidden>
|
||||
Tìm theo số báo danh hoặc họ tên
|
||||
</label>
|
||||
<input
|
||||
id="search-input"
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder="Nhập SBD hoặc họ tên (VD: nguyen van a)..."
|
||||
disabled={disabled}
|
||||
autoFocus
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
{query && (
|
||||
<button
|
||||
type="button"
|
||||
className="search-clear"
|
||||
onClick={() => setQuery("")}
|
||||
aria-label="Xóa từ khóa"
|
||||
tabIndex={-1}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<button type="submit" disabled={disabled || !query.trim()}>
|
||||
Tra cứu
|
||||
</button>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import initSqlJs from "sql.js";
|
||||
|
||||
const SQL_WASM_URL = "https://sql.js.org/dist/sql-wasm.wasm";
|
||||
|
||||
/**
|
||||
* Hook to load a SQLite database from a gzipped URL into sql.js.
|
||||
* Returns { db, loading, error, progress }.
|
||||
*/
|
||||
export function useSqlite(dbUrl) {
|
||||
const [db, setDb] = useState(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(null);
|
||||
const [progress, setProgress] = useState(0);
|
||||
const dbRef = useRef(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const SQL = await initSqlJs({ locateFile: () => SQL_WASM_URL });
|
||||
|
||||
const response = await fetch(dbUrl);
|
||||
if (!response.ok)
|
||||
throw new Error(`Failed to fetch database: ${response.status}`);
|
||||
|
||||
const contentLength = +response.headers.get("Content-Length") || 0;
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let received = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
received += value.length;
|
||||
if (contentLength > 0) {
|
||||
setProgress(Math.round((received / contentLength) * 100));
|
||||
}
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const blob = new Blob(chunks);
|
||||
let arrayBuffer;
|
||||
|
||||
if (dbUrl.endsWith(".gz")) {
|
||||
const ds = new DecompressionStream("gzip");
|
||||
const decompressed = blob.stream().pipeThrough(ds);
|
||||
const decompressedBlob = await new Response(decompressed).blob();
|
||||
arrayBuffer = await decompressedBlob.arrayBuffer();
|
||||
} else {
|
||||
arrayBuffer = await blob.arrayBuffer();
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const database = new SQL.Database(new Uint8Array(arrayBuffer));
|
||||
dbRef.current = database;
|
||||
setDb(database);
|
||||
setLoading(false);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
setError(err.message);
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
load();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (dbRef.current) {
|
||||
dbRef.current.close();
|
||||
dbRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [dbUrl]);
|
||||
|
||||
return { db, loading, error, progress };
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
:root {
|
||||
/* Light theme tokens */
|
||||
--bg: #f7f8fb;
|
||||
--surface: #ffffff;
|
||||
--surface-alt: #f2f4f9;
|
||||
--border: #e3e6ee;
|
||||
--border-strong: #cfd4e1;
|
||||
--text: #1a1a2e;
|
||||
--text-muted: #5c6275;
|
||||
--text-faint: #8a8fa3;
|
||||
--accent: #4361ee;
|
||||
--accent-hover: #3a56d4;
|
||||
--accent-soft: #e9edff;
|
||||
--danger: #d32f2f;
|
||||
--danger-bg: #fdecea;
|
||||
--warn: #b45309;
|
||||
--warn-bg: #fff4d6;
|
||||
--ok: #0f8a4f;
|
||||
--ok-bg: #e4f7ec;
|
||||
--shadow-sm: 0 1px 2px rgba(20, 22, 38, 0.06);
|
||||
--shadow-md: 0 4px 14px rgba(20, 22, 38, 0.08);
|
||||
--radius: 10px;
|
||||
--radius-sm: 6px;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #181b24;
|
||||
--surface-alt: #21252f;
|
||||
--border: #2a2f3d;
|
||||
--border-strong: #3a4052;
|
||||
--text: #eef0f6;
|
||||
--text-muted: #a7adbe;
|
||||
--text-faint: #6e7486;
|
||||
--accent: #8aa1ff;
|
||||
--accent-hover: #a6b8ff;
|
||||
--accent-soft: #1e2647;
|
||||
--danger: #ff7a7a;
|
||||
--danger-bg: #3a1d1d;
|
||||
--warn: #ffc875;
|
||||
--warn-bg: #3a2a12;
|
||||
--ok: #6ee2a4;
|
||||
--ok-bg: #133524;
|
||||
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.5);
|
||||
--shadow-md: 0 4px 14px rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
}
|
||||
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
color-scheme: light dark;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font-family:
|
||||
"Inter", system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
:focus-visible {
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import "./index.css";
|
||||
import App from "./App.jsx";
|
||||
|
||||
createRoot(document.getElementById("root")).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>,
|
||||
);
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-sync xlsxread Rust source from thptqg2017.
|
||||
#
|
||||
# Source SHA: 8b4a755c115595bf1b937d749eb1133efb3a6e22 (chore/xlsxread-rust)
|
||||
#
|
||||
# Re-run this when xlsxread is updated upstream. The configs/ directory is
|
||||
# NOT synced — it contains thptqg2016-specific configs and test stubs that
|
||||
# must be maintained here independently.
|
||||
#
|
||||
# Usage:
|
||||
# ./tools/sync-from-thptqg2017.sh /path/to/thptqg2017
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SRC="${1:?Usage: $0 /path/to/thptqg2017}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEST="$SCRIPT_DIR/xlsxread"
|
||||
|
||||
if [ ! -d "$SRC/tools/xlsxread" ]; then
|
||||
echo "Error: $SRC/tools/xlsxread not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Syncing from: $SRC/tools/xlsxread"
|
||||
echo "Syncing to: $DEST"
|
||||
echo ""
|
||||
|
||||
# Sync source, tests, and Cargo manifests — exclude build artifacts and dataset configs
|
||||
for item in src tests Cargo.toml Cargo.lock; do
|
||||
if [ -e "$SRC/tools/xlsxread/$item" ]; then
|
||||
cp -r "$SRC/tools/xlsxread/$item" "$DEST/"
|
||||
echo " synced: $item"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Sync complete."
|
||||
echo "Next steps:"
|
||||
echo " 1. Review src/ for breaking changes to config.rs / writer.rs that"
|
||||
echo " may affect format_detect_2016.rs or the thptqg2016-data.toml config."
|
||||
echo " 2. Rebuild: cargo build --release --manifest-path $DEST/Cargo.toml"
|
||||
echo " 3. Test: cargo test --manifest-path $DEST/Cargo.toml"
|
||||
echo " 4. Commit on a chore/xlsxread-sync-... branch."
|
||||
@@ -1,7 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: "/thptqg/2016/",
|
||||
});
|
||||
@@ -1,27 +0,0 @@
|
||||
.idea
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
### Node ###
|
||||
node_modules/
|
||||
|
||||
### Generated databases (all 3 variants) ###
|
||||
public/thptqg2017.db
|
||||
public/thptqg2017.db.gz
|
||||
public-old/thptqg2017.db
|
||||
public-old/thptqg2017.db.gz
|
||||
public-old2/thptqg2017.db
|
||||
public-old2/thptqg2017.db.gz
|
||||
|
||||
### Build output ###
|
||||
dist/
|
||||
|
||||
### Rust build artefacts ###
|
||||
tools/xlsxread/target/
|
||||
-201
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
# Vietnam THPT 2017 Score Lookup
|
||||
|
||||
Static React + SQLite site for looking up Vietnamese high school graduation exam scores (2017). The full database (~861k students, 63 provinces) ships to the browser as a compressed SQLite file and queries run client-side via `sql.js` — no backend.
|
||||
|
||||
## Live sites
|
||||
|
||||
Three deployments, one per dataset:
|
||||
|
||||
| URL | Data source | Rows |
|
||||
|---|---|---|
|
||||
| `/thptqg2017/` | `data/` — [baotintuc.vn](https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm) CDN, `.xls` | **861,068** |
|
||||
| `/thptqg2017/old/` | `data-old/` — earlier `.xlsx` collection from Vietnamese news sites at the time (exact origin no longer known) | 847,348 |
|
||||
| `/thptqg2017/old2/` | `data-old2/` — partial re-export (54 provinces) also from contemporary news sites, exact origin unrecorded | 679,764 |
|
||||
|
||||
## Features
|
||||
|
||||
- Diacritics-insensitive name search (`"nguyen"` matches `"Nguyễn"`)
|
||||
- Live debounced search with URL deep-link (`?q=49008235`)
|
||||
- Single-result detail card with per-subject TFT rarity-tiered scores (6 tiers, ≤1 → 9-10) and all 49 admission blocks (A00 – D15)
|
||||
- Share button: copies a formatted summary plus deep-link URL
|
||||
- SQL query tab with grouped presets (rankings, Long An filters, statistics, schema)
|
||||
- Light + dark mode (follows OS preference)
|
||||
- Keyboard shortcut `/` to focus search
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js 24+
|
||||
- pnpm
|
||||
- Rust (stable) — installed via [rustup](https://rustup.rs/); required for `build:db*` scripts
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
pnpm build:db # compile xlsxread + parse data/ → public/thptqg2017.db (~2 min, 159 MB)
|
||||
gzip -kf -9 public/thptqg2017.db
|
||||
pnpm dev # http://localhost:5173
|
||||
```
|
||||
|
||||
The database build pipeline uses the `xlsxread` Rust CLI located at
|
||||
`tools/xlsxread/`. It reads `.xls`/`.xlsx` source files, strips
|
||||
diacritics, parses score text, and writes a SQLite database — replacing
|
||||
the former Node.js + `xlsx` (SheetJS) pipeline. See
|
||||
`tools/xlsxread/README.md` for CLI invocation details and config schema.
|
||||
|
||||
## Scripts
|
||||
|
||||
| Command | Action |
|
||||
|---|---|
|
||||
| `pnpm dev` | Vite dev server |
|
||||
| `pnpm build` | Production build (main variant → `dist/`) |
|
||||
| `pnpm build:old` / `build:old2` | Build variant sites to `dist/old/`, `dist/old2/` |
|
||||
| `pnpm build:all` | All 3 web variants |
|
||||
| `pnpm build:rust` | Compile `tools/xlsxread` release binary (run once; auto-called by `build:db*`) |
|
||||
| `pnpm build:db` | Build main DB from `data/` via xlsxread |
|
||||
| `pnpm build:db:old` / `build:db:old2` | Build old / old2 variant DBs via xlsxread |
|
||||
| `pnpm build:db:all` | All 3 DBs via xlsxread |
|
||||
| `pnpm lint` | ESLint |
|
||||
| `node scripts/crawl-baotintuc.js` | Re-download all 63 province files from baotintuc.vn |
|
||||
| `node scripts/check-duplicates.js` | MD5 + row-content duplicate audit |
|
||||
| `node scripts/diff-datasets.js` | Compare `public/` vs `backup/` DB (when backup present) |
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
.
|
||||
├── data/ # 63 .xls files (source)
|
||||
├── data-old/ # 63 .xlsx (previous export)
|
||||
├── data-old2/ # 54 .xlsx (update/ overrides)
|
||||
├── public/ # main variant assets + thptqg2017.db.gz
|
||||
├── public-old/ # old variant assets
|
||||
├── public-old2/ # old2 variant assets
|
||||
├── scripts/
|
||||
│ ├── crawl-baotintuc.js # downloader
|
||||
│ ├── check-duplicates.js # md5 dup detector
|
||||
│ └── diff-datasets.js # DB-to-DB comparator
|
||||
├── src/
|
||||
│ ├── App.jsx
|
||||
│ ├── App.css
|
||||
│ ├── components/{search-form, score-table, student-detail, custom-query}.jsx
|
||||
│ ├── hooks/use-sqlite.js
|
||||
│ └── lib/admission-blocks.js # 49 admission-block definitions + score-tier helper
|
||||
├── tools/
|
||||
│ └── xlsxread/ # Rust CLI — reads .xls/.xlsx, writes SQLite
|
||||
│ ├── configs/
|
||||
│ │ ├── thptqg2017-data.toml # config for data/ (63 .xls, all sheets)
|
||||
│ │ ├── thptqg2017-data-old.toml # config for data-old/ (63 .xlsx, sheet 0)
|
||||
│ │ └── thptqg2017-data-old2.toml # config for data-old2/ (54 .xlsx, all sheets)
|
||||
│ ├── src/ # Rust source
|
||||
│ └── README.md # CLI reference + config schema
|
||||
├── docs/ # see docs/README.md
|
||||
├── index.html
|
||||
├── vite.config.js
|
||||
└── package.json
|
||||
```
|
||||
|
||||
See `docs/` for architecture + deployment details.
|
||||
|
||||
## License
|
||||
|
||||
See `LICENSE`.
|
||||
|
||||
## Source
|
||||
|
||||
- **`data/`** — scraped in full from the baotintuc.vn 2017 announcement article:
|
||||
`https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm`
|
||||
Reproducible via `node scripts/crawl-baotintuc.js`.
|
||||
- **`data-old/`** and **`data-old2/`** — collected from Vietnamese news sites at the
|
||||
time of the 2017 exam. The specific publisher URLs were not recorded and cannot
|
||||
be recovered now. These datasets are preserved for comparison and historical
|
||||
reference only; `data/` is the canonical source for the main deployment.
|
||||
|
||||
Intended for reference only.
|
||||
@@ -1,19 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://fonts.googleapis.com/css2?family=Be+Vietnam+Pro:wght@400;500;600;700&display=swap"
|
||||
/>
|
||||
<title>Tra cứu điểm thi THPT QG 2017</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"name": "thptqg2017",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"description": "Tra cứu điểm thi THPT QG 2017",
|
||||
"scripts": {
|
||||
"build:rust": "cargo build --release --manifest-path tools/xlsxread/Cargo.toml",
|
||||
"build:db": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data.toml --input data --output public/thptqg2017.db",
|
||||
"build:db:old": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old.toml --input data-old --output public-old/thptqg2017.db",
|
||||
"build:db:old2": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old2.toml --input data-old2 --output public-old2/thptqg2017.db",
|
||||
"build:db:all": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data.toml --input data --output public/thptqg2017.db && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old.toml --input data-old --output public-old/thptqg2017.db && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old2.toml --input data-old2 --output public-old2/thptqg2017.db",
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"build:old": "node -e \"process.env.VARIANT='old';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:old2": "node -e \"process.env.VARIANT='old2';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:all": "pnpm build && pnpm build:old && pnpm build:old2",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint ."
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/tiennm99/thptqg2017.git"
|
||||
},
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"sql.js": "^1.14.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.5.2",
|
||||
"globals": "^17.4.0",
|
||||
"vite": "^8.0.16"
|
||||
}
|
||||
}
|
||||
Generated
-1624
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
allowBuilds:
|
||||
better-sqlite3: true
|
||||
Generated
-1159
File diff suppressed because it is too large
Load Diff
@@ -1,30 +0,0 @@
|
||||
[package]
|
||||
name = "xlsxread"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016"
|
||||
|
||||
[dependencies]
|
||||
calamine = "0.26"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
toml = "0.8"
|
||||
regex = "1"
|
||||
unicode-normalization = "0.1"
|
||||
thiserror = "1"
|
||||
anyhow = "1"
|
||||
glob = "0.3"
|
||||
|
||||
# zip is already a transitive dep of calamine; pin explicitly so tests can use it
|
||||
[dev-dependencies]
|
||||
zip = "2"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
[[bin]]
|
||||
name = "xlsxread"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[test]]
|
||||
name = "golden"
|
||||
path = "tests/golden.rs"
|
||||
@@ -1,75 +0,0 @@
|
||||
# Config for data-old/ — 63 .xlsx files (pre-baotintuc refresh)
|
||||
# Sheet mode: "first" — single-sheet workbooks, never hit 65k row cap
|
||||
# SBD validation: require ^\d+$ (build-database-old.js:55 guard)
|
||||
# Blank row strip: off (no explicit blank-skip in build-database-old.js)
|
||||
|
||||
[reader]
|
||||
sheet_mode = "first"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,76 +0,0 @@
|
||||
# Config for data-old2/ — 54 .xlsx files (corrected-export set)
|
||||
# Sheet mode: "all" — HCM (24.HCM_UTLQ.xlsx) overflows into Sheet2 (+6,446 rows)
|
||||
# SBD validation: require ^\d+$ (build-database-old2.js:57 guard)
|
||||
# Blank row strip: true — skip fully blank rows BEFORE counting sourceRows
|
||||
# (build-database-old2.js:50-51: blank row check before sourceRows++)
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = true
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,75 +0,0 @@
|
||||
# Config for data/ — 63 .xls files from baotintuc.vn
|
||||
# Sheet mode: "all" because Hà Nội and HCM overflow into Sheet2 (65k row cap)
|
||||
# SBD validation: no numeric guard (build-database.js does not apply ^\d+$)
|
||||
# Blank row strip: off
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,174 +0,0 @@
|
||||
/// Audit subcommand: replicates audit-row-counts.js exactly.
|
||||
///
|
||||
/// Reads all .xlsx files from the input directory (sheet 0 only, matching the
|
||||
/// JS script's behaviour at audit-row-counts.js:33), collects distinct SBDs
|
||||
/// into a HashSet, then queries `SELECT COUNT(*) FROM student` from the DB.
|
||||
/// Prints the same lines as audit-row-counts.js:54-62 and exits 0 on match,
|
||||
/// 1 on mismatch.
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader};
|
||||
|
||||
use crate::config::DatasetConfig;
|
||||
use crate::error::BuildError;
|
||||
use crate::reader::is_header_row;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audit result
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct AuditResult {
|
||||
pub total_data_rows: u64,
|
||||
pub both_empty: u64,
|
||||
pub empty_name: u64,
|
||||
pub empty_sbd: u64,
|
||||
pub distinct_sbds: usize,
|
||||
pub db_count: i64,
|
||||
pub matched: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main audit logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Collect distinct SBDs from all xlsx files in `input_dir`, query `db_path`,
|
||||
/// print the audit report and return the result.
|
||||
///
|
||||
/// The JS script reads only sheet 0 for every file (audit-row-counts.js:33).
|
||||
/// Unlike build-database.js, the audit script does NOT iterate all sheets.
|
||||
pub fn run_audit(
|
||||
input_dir: &Path,
|
||||
db_path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
) -> Result<AuditResult, BuildError> {
|
||||
// Collect .xlsx files (audit-row-counts.js only checks .xlsx — line 15)
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.map_err(|e| BuildError::Io {
|
||||
path: input_dir.display().to_string(),
|
||||
source: e,
|
||||
})?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.eq_ignore_ascii_case("xlsx"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let mut all_sbd: HashSet<String> = HashSet::new();
|
||||
let mut total_data_rows: u64 = 0;
|
||||
let mut empty_name: u64 = 0;
|
||||
let mut empty_sbd: u64 = 0;
|
||||
let mut both_empty: u64 = 0;
|
||||
|
||||
for file in &files {
|
||||
let path_str = file.display().to_string();
|
||||
let mut workbook = open_workbook_auto(file).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// audit-row-counts.js reads only sheet 0 (line 33: wb.SheetNames[0])
|
||||
let range =
|
||||
workbook
|
||||
.worksheet_range(&sheet_names[0])
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
for raw in range.rows() {
|
||||
let row: Vec<Data> = raw.to_vec();
|
||||
|
||||
// Skip header row on first row only
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
total_data_rows += 1;
|
||||
|
||||
let ho_ten = row
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let sbd = row
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
if ho_ten.is_empty() && sbd.is_empty() {
|
||||
both_empty += 1;
|
||||
continue;
|
||||
}
|
||||
if ho_ten.is_empty() {
|
||||
empty_name += 1;
|
||||
}
|
||||
if sbd.is_empty() {
|
||||
empty_sbd += 1;
|
||||
}
|
||||
if !sbd.is_empty() {
|
||||
all_sbd.insert(sbd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query DB count
|
||||
let conn =
|
||||
rusqlite::Connection::open_with_flags(db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let distinct_sbds = all_sbd.len();
|
||||
let matched = distinct_sbds as i64 == db_count;
|
||||
|
||||
Ok(AuditResult {
|
||||
total_data_rows,
|
||||
both_empty,
|
||||
empty_name,
|
||||
empty_sbd,
|
||||
distinct_sbds,
|
||||
db_count,
|
||||
matched,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Print audit report — mirrors audit-row-counts.js:54-62 exactly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn print_audit_report(r: &AuditResult) {
|
||||
println!("=== Source vs DB ===");
|
||||
println!(
|
||||
"Source: total data rows across all files: {}",
|
||||
r.total_data_rows
|
||||
);
|
||||
println!(
|
||||
"Source: rows with empty name AND sbd (skipped): {}",
|
||||
r.both_empty
|
||||
);
|
||||
println!("Source: rows with missing name only: {}", r.empty_name);
|
||||
println!("Source: rows with missing sbd only: {}", r.empty_sbd);
|
||||
println!("Source: distinct SBDs: {}", r.distinct_sbds);
|
||||
println!("DB: row count: {}", r.db_count);
|
||||
println!(
|
||||
"Match: {}",
|
||||
if r.matched {
|
||||
"YES — all unique SBDs accounted for".to_string()
|
||||
} else {
|
||||
format!("NO — gap of {}", r.distinct_sbds as i64 - r.db_count)
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
/// CLI argument structs via clap derive.
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "xlsxread",
|
||||
version,
|
||||
about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets"
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[command(subcommand)]
|
||||
pub cmd: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum Cmd {
|
||||
/// Read input spreadsheets and write a SQLite database
|
||||
Build {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xls / .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// Output SQLite database path
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
|
||||
/// Audit: compare distinct SBD count from xlsx files vs DB row count
|
||||
Audit {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// SQLite database to compare against
|
||||
#[arg(long)]
|
||||
db: PathBuf,
|
||||
},
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Top-level dataset configuration loaded from a .toml file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct DatasetConfig {
|
||||
pub reader: ReaderCfg,
|
||||
pub columns: ColumnMap,
|
||||
pub validation: ValidationCfg,
|
||||
pub header: HeaderCfg,
|
||||
pub schema: SchemaCfg,
|
||||
/// field name → regex source string (one entry per scoreable subject)
|
||||
pub scores: HashMap<String, String>,
|
||||
pub insert: InsertCfg,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ReaderCfg {
|
||||
/// "all" → iterate every sheet (handles HCM/HN overflow); "first" → sheet 0 only
|
||||
pub sheet_mode: SheetMode,
|
||||
/// If true, skip rows where every cell is empty/null before counting (data-old2 quirk)
|
||||
pub strip_blank_rows: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SheetMode {
|
||||
All,
|
||||
First,
|
||||
}
|
||||
|
||||
/// Zero-indexed column positions in the source spreadsheet row.
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ColumnMap {
|
||||
pub ho_ten: usize,
|
||||
pub ngay_sinh: usize,
|
||||
pub so_bao_danh: usize,
|
||||
pub diem_thi: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ValidationCfg {
|
||||
/// build-database-old.js / -old2.js require soBaoDanh to match ^\d+$
|
||||
pub require_numeric_sbd: bool,
|
||||
pub require_nonempty_name: bool,
|
||||
pub require_nonempty_sbd: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct HeaderCfg {
|
||||
/// Tokens to match against row[0].to_uppercase() to detect a header row
|
||||
pub tokens: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct SchemaCfg {
|
||||
/// DDL executed verbatim before inserts (CREATE TABLE + CREATE INDEX)
|
||||
pub ddl: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InsertCfg {
|
||||
/// Parameterised INSERT OR REPLACE SQL using :named_param style
|
||||
pub sql: String,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Loader
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn load_config(path: &Path) -> Result<DatasetConfig, BuildError> {
|
||||
let text = fs::read_to_string(path).map_err(|e| BuildError::Io {
|
||||
path: path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
let cfg: DatasetConfig = toml::from_str(&text)?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_TOML: &str = r#"
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = "CREATE TABLE student (so_bao_danh TEXT PRIMARY KEY);"
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = "INSERT OR REPLACE INTO student (so_bao_danh) VALUES (:so_bao_danh)"
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn config_round_trip() {
|
||||
let cfg: DatasetConfig = toml::from_str(SAMPLE_TOML).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::All);
|
||||
assert!(!cfg.reader.strip_blank_rows);
|
||||
assert_eq!(cfg.columns.ho_ten, 0);
|
||||
assert_eq!(cfg.columns.diem_thi, 3);
|
||||
assert!(!cfg.validation.require_numeric_sbd);
|
||||
assert!(cfg.validation.require_nonempty_name);
|
||||
assert_eq!(cfg.header.tokens.len(), 3);
|
||||
assert!(cfg.scores.contains_key("toan"));
|
||||
assert!(cfg.scores.contains_key("ngu_van"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_first_sheet_mode() {
|
||||
let toml_str = SAMPLE_TOML.replace(r#"sheet_mode = "all""#, r#"sheet_mode = "first""#);
|
||||
let cfg: DatasetConfig = toml::from_str(&toml_str).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::First);
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum BuildError {
|
||||
#[error("I/O error for {path}: {source}")]
|
||||
Io {
|
||||
path: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
|
||||
#[error("Calamine error for {path}: {source}")]
|
||||
Calamine {
|
||||
path: String,
|
||||
#[source]
|
||||
source: calamine::Error,
|
||||
},
|
||||
|
||||
#[error("SQLite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
|
||||
#[error("Config parse error: {0}")]
|
||||
Config(#[from] toml::de::Error),
|
||||
|
||||
#[error("Regex compile error for pattern '{pattern}': {source}")]
|
||||
Regex {
|
||||
pattern: String,
|
||||
#[source]
|
||||
source: regex::Error,
|
||||
},
|
||||
|
||||
#[error("Schema has no sheets in file: {0}")]
|
||||
NoSheets(String),
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
/// Public library interface for integration tests.
|
||||
/// The binary entry point is src/main.rs; this file re-exports the internal
|
||||
/// modules so tests/golden.rs can call them without going through the CLI.
|
||||
pub mod audit;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod reader;
|
||||
pub mod transform;
|
||||
pub mod writer;
|
||||
@@ -1,193 +0,0 @@
|
||||
/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts.
|
||||
///
|
||||
/// Subcommands:
|
||||
/// build — read .xls/.xlsx files → write SQLite DB
|
||||
/// audit — compare distinct SBD count from xlsx vs DB row count
|
||||
///
|
||||
/// Library modules are declared in lib.rs; main.rs only adds the CLI layer.
|
||||
mod cli;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
|
||||
use cli::{Cli, Cmd};
|
||||
use xlsxread::audit;
|
||||
use xlsxread::config::load_config;
|
||||
use xlsxread::reader::{is_all_blank, process_file};
|
||||
use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason};
|
||||
use xlsxread::writer::{finish_db, insert_row, open_db, SCORE_FIELDS};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.cmd {
|
||||
Cmd::Build {
|
||||
schema,
|
||||
input,
|
||||
output,
|
||||
} => {
|
||||
run_build(&schema, &input, &output)?;
|
||||
}
|
||||
Cmd::Audit { schema, input, db } => {
|
||||
let cfg = load_config(&schema)
|
||||
.with_context(|| format!("Failed to load config: {}", schema.display()))?;
|
||||
let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?;
|
||||
audit::print_audit_report(&result);
|
||||
if !result.matched {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build subcommand
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> {
|
||||
let cfg = load_config(schema_path)
|
||||
.with_context(|| format!("Failed to load config: {}", schema_path.display()))?;
|
||||
|
||||
// Compile score regexes once at startup
|
||||
let patterns =
|
||||
CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?;
|
||||
|
||||
// Collect input files (.xls and .xlsx), sorted for deterministic order
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.with_context(|| format!("Cannot read input dir: {}", input_dir.display()))?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let lower = e.to_lowercase();
|
||||
lower == "xls" || lower == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let dataset_label = input_dir
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("data");
|
||||
|
||||
println!(
|
||||
"[build] {dataset_label}/ → {} ({} files)",
|
||||
output_path.display(),
|
||||
files.len()
|
||||
);
|
||||
|
||||
// Open (or recreate) DB and apply DDL
|
||||
let conn = open_db(output_path, &cfg)
|
||||
.with_context(|| format!("Failed to open DB: {}", output_path.display()))?;
|
||||
|
||||
let mut total_source_rows: u64 = 0;
|
||||
let mut total_skipped: u64 = 0;
|
||||
let mut total_errors: u64 = 0;
|
||||
|
||||
let is_old2 = dataset_label.contains("old2");
|
||||
let strip_blank = cfg.reader.strip_blank_rows;
|
||||
|
||||
// Single transaction over all files — mirrors the Node `db.transaction(() => { ... })()`
|
||||
conn.execute_batch("BEGIN")?;
|
||||
|
||||
for file in &files {
|
||||
let base = file
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
let mut file_rows: u64 = 0;
|
||||
let mut file_skipped: u64 = 0;
|
||||
let mut file_errors: u64 = 0;
|
||||
|
||||
let process_result = process_file(file, &cfg, |_sheet_idx, raw| {
|
||||
// data-old2: skip fully blank rows BEFORE counting sourceRows
|
||||
let all_blank = is_all_blank(raw);
|
||||
if strip_blank && all_blank {
|
||||
return;
|
||||
}
|
||||
|
||||
total_source_rows += 1;
|
||||
|
||||
let ho_ten = raw
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
match validate_row(
|
||||
&ho_ten,
|
||||
&so_bao_danh,
|
||||
&cfg.validation,
|
||||
strip_blank,
|
||||
all_blank,
|
||||
) {
|
||||
Err(SkipReason::BlankRow) => {
|
||||
// Already guarded above; won't reach here
|
||||
}
|
||||
Err(_) => {
|
||||
file_skipped += 1;
|
||||
return;
|
||||
}
|
||||
Ok(()) => {}
|
||||
}
|
||||
|
||||
let parsed = xlsxread::transform::transform_row(raw, &cfg, &patterns);
|
||||
|
||||
match insert_row(&conn, &cfg.insert.sql, &parsed, SCORE_FIELDS) {
|
||||
Ok(()) => {
|
||||
file_rows += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
file_errors += 1;
|
||||
if total_errors + file_errors <= 5 {
|
||||
eprintln!(" [warn] {base}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
match process_result {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!(" [error] {base}: {e}");
|
||||
file_errors += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total_skipped += file_skipped;
|
||||
total_errors += file_errors;
|
||||
|
||||
// Per-file row count line — mirrors `console.log(` ${base}: ${fileRows} rows`)`
|
||||
println!(" {base}: {file_rows} rows");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT")?;
|
||||
|
||||
// VACUUM + stats output
|
||||
finish_db(
|
||||
&conn,
|
||||
output_path,
|
||||
total_source_rows,
|
||||
total_skipped,
|
||||
total_errors,
|
||||
dataset_label,
|
||||
files.len(),
|
||||
is_old2,
|
||||
)
|
||||
.with_context(|| "Failed to finalise DB")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
/// Spreadsheet reader: wraps calamine to iterate rows across sheets.
|
||||
///
|
||||
/// Sheet selection mirrors the JS scripts:
|
||||
/// - sheet_mode = "all" → iterate every sheet (handles HCM/HN 65k overflow in data/)
|
||||
/// - sheet_mode = "first" → sheet 0 only (data-old/)
|
||||
///
|
||||
/// Header detection mirrors build-lib.js isHeaderRow:
|
||||
/// row[0].toUpperCase() in {"HO_TEN", "HỌ TÊN", "STT"}
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader, Sheets};
|
||||
|
||||
use crate::config::{DatasetConfig, HeaderCfg, SheetMode};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public row representation from calamine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub type RawRow = Vec<Data>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Header detection — mirrors build-lib.js isHeaderRow
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns true when the first cell (uppercased) matches one of the configured
|
||||
/// header tokens. Used to skip the header row on the first row of each sheet.
|
||||
pub fn is_header_row(row: &[Data], header_cfg: &HeaderCfg) -> bool {
|
||||
if row.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
let first = row[0].to_string().trim().to_uppercase();
|
||||
header_cfg.tokens.iter().any(|t| t.to_uppercase() == first)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// All-blank row check (data-old2: strip_blank_rows)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn is_all_blank(row: &[Data]) -> bool {
|
||||
row.iter()
|
||||
.all(|c| matches!(c, Data::Empty) || c.to_string().trim().is_empty())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// File processor — yields all data rows from the file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Process one spreadsheet file, calling `on_row` for each data row.
|
||||
///
|
||||
/// `on_row` receives `(sheet_index, row_index_in_sheet, raw_row)` where
|
||||
/// `row_index_in_sheet` is 0-based AFTER the header has been consumed.
|
||||
/// Returns `(sheets_seen, total_rows_yielded)`.
|
||||
pub fn process_file<F>(
|
||||
path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
mut on_row: F,
|
||||
) -> Result<(usize, usize), BuildError>
|
||||
where
|
||||
F: FnMut(usize, &RawRow),
|
||||
{
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
// calamine::open_workbook_auto dispatches on file extension
|
||||
let mut workbook: Sheets<_> = open_workbook_auto(path).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names: Vec<String> = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
return Err(BuildError::NoSheets(path_str.clone()));
|
||||
}
|
||||
|
||||
// Sheet selection per config
|
||||
let sheets_to_read: Vec<String> = match cfg.reader.sheet_mode {
|
||||
SheetMode::All => sheet_names.clone(),
|
||||
SheetMode::First => vec![sheet_names[0].clone()],
|
||||
};
|
||||
|
||||
let mut total_rows = 0usize;
|
||||
|
||||
for (sheet_idx, sheet_name) in sheets_to_read.iter().enumerate() {
|
||||
let range = workbook
|
||||
.worksheet_range(sheet_name)
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
|
||||
for raw in range.rows() {
|
||||
let row: RawRow = raw.to_vec();
|
||||
|
||||
// Skip header row on first row of each sheet (matches JS: `if (i === 0 && isHeaderRow(...))`)
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
on_row(sheet_idx, &row);
|
||||
total_rows += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((sheets_to_read.len(), total_rows))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::HeaderCfg;
|
||||
|
||||
fn hdr(tokens: &[&str]) -> HeaderCfg {
|
||||
HeaderCfg {
|
||||
tokens: tokens.iter().map(|s| s.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten() {
|
||||
let row = vec![
|
||||
Data::String("HO_TEN".into()),
|
||||
Data::String("NGAY_SINH".into()),
|
||||
Data::String("SBD".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_stt() {
|
||||
let row = vec![
|
||||
Data::String("STT".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten_unicode() {
|
||||
let row = vec![
|
||||
Data::String("HỌ TÊN".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_data_row() {
|
||||
let row = vec![
|
||||
Data::String("Nguyen Van A".into()),
|
||||
Data::String("01/01/2000".into()),
|
||||
Data::String("12345678".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_short_row() {
|
||||
let row = vec![Data::String("HO_TEN".into()), Data::Empty];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_case_insensitive() {
|
||||
let row = vec![
|
||||
Data::String("ho_ten".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_row_detection() {
|
||||
let row = vec![Data::Empty, Data::Empty, Data::String("".into())];
|
||||
assert!(is_all_blank(&row));
|
||||
|
||||
let row2 = vec![Data::String("Nguyen".into()), Data::Empty, Data::Empty];
|
||||
assert!(!is_all_blank(&row2));
|
||||
}
|
||||
}
|
||||
@@ -1,398 +0,0 @@
|
||||
/// Row transformation: ascii normalisation, score regex parsing, validation.
|
||||
///
|
||||
/// `to_ascii` replicates build-lib.js `toAscii` exactly:
|
||||
/// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase()
|
||||
use std::collections::HashMap;
|
||||
|
||||
use regex::Regex;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::config::{DatasetConfig, ValidationCfg};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compiled score patterns (built once at startup from config)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct CompiledPatterns {
|
||||
/// Ordered list so INSERT column order is deterministic
|
||||
pub patterns: Vec<(String, Regex)>,
|
||||
}
|
||||
|
||||
impl CompiledPatterns {
|
||||
pub fn new(scores: &HashMap<String, String>) -> Result<Self, BuildError> {
|
||||
let mut patterns = Vec::with_capacity(scores.len());
|
||||
for (field, src) in scores {
|
||||
let re = Regex::new(src).map_err(|e| BuildError::Regex {
|
||||
pattern: src.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
patterns.push((field.clone(), re));
|
||||
}
|
||||
// Sort for deterministic order across HashMap iteration
|
||||
patterns.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
Ok(Self { patterns })
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// to_ascii — must be byte-for-byte equivalent to build-lib.js toAscii
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalise a Vietnamese name to an ASCII slug.
|
||||
///
|
||||
/// Algorithm mirrors the JavaScript `toAscii` in build-lib.js:
|
||||
/// 1. NFD decompose (splits base + combining diacritics)
|
||||
/// 2. Drop all Unicode combining marks (U+0300–U+036F)
|
||||
/// 3. Replace đ/Đ with d (NFD does not decompose đ)
|
||||
/// 4. Lowercase
|
||||
pub fn to_ascii(s: &str) -> String {
|
||||
// Step 1 + 2: NFD then filter out combining marks (Unicode category M)
|
||||
let decomposed: String = s
|
||||
.nfd()
|
||||
.filter(|c| !('\u{0300}'..='\u{036f}').contains(c))
|
||||
.collect();
|
||||
|
||||
// Step 3: đ/Đ are not decomposed by NFD — replace explicitly
|
||||
let replaced = decomposed.replace(['đ', 'Đ'], "d");
|
||||
|
||||
// Step 4: lowercase
|
||||
replaced.to_lowercase()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsed row ready for DB insert
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct ParsedRow {
|
||||
pub so_bao_danh: String,
|
||||
pub ho_ten: String,
|
||||
pub ho_ten_ascii: String,
|
||||
pub ngay_sinh: Option<String>,
|
||||
/// Subject field → float value; absent subjects not in map → NULL
|
||||
pub scores: HashMap<String, f64>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Row validation — mirrors the per-script skip logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns `None` when the row should be skipped entirely (before sourceRows counter).
|
||||
/// Returns `Some(reason)` when the row should be counted as sourceRows but skipped.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum SkipReason {
|
||||
/// Row is fully blank (data-old2 only, before sourceRows counter)
|
||||
BlankRow,
|
||||
/// soBaoDanh or hoTen empty/missing
|
||||
EmptyField,
|
||||
/// soBaoDanh contains non-digit characters (data-old / data-old2 guard)
|
||||
NonNumericSbd,
|
||||
}
|
||||
|
||||
/// Validates a raw cell slice against the dataset's `ValidationCfg`.
|
||||
/// Returns `Ok(())` on pass, `Err(SkipReason)` on fail.
|
||||
pub fn validate_row(
|
||||
ho_ten: &str,
|
||||
so_bao_danh: &str,
|
||||
cfg: &ValidationCfg,
|
||||
strip_blank_rows: bool,
|
||||
all_blank: bool,
|
||||
) -> Result<(), SkipReason> {
|
||||
// data-old2: skip fully blank rows BEFORE counting sourceRows
|
||||
if strip_blank_rows && all_blank {
|
||||
return Err(SkipReason::BlankRow);
|
||||
}
|
||||
|
||||
if cfg.require_nonempty_sbd && so_bao_danh.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_nonempty_name && ho_ten.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_numeric_sbd && !so_bao_danh.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(SkipReason::NonNumericSbd);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Score parsing — mirrors build-lib.js parseScores
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse a DIEM_THI cell string and extract matching subject scores.
|
||||
pub fn parse_scores(diem_thi: &str, patterns: &CompiledPatterns) -> HashMap<String, f64> {
|
||||
let mut out = HashMap::new();
|
||||
for (field, re) in &patterns.patterns {
|
||||
if let Some(caps) = re.captures(diem_thi) {
|
||||
if let Some(m) = caps.get(1) {
|
||||
if let Ok(v) = m.as_str().parse::<f64>() {
|
||||
if v.is_finite() {
|
||||
out.insert(field.clone(), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Full row transform
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract and transform one spreadsheet row into a `ParsedRow`.
|
||||
/// `raw` is the full cell slice; column indices come from `cfg.columns`.
|
||||
pub fn transform_row(
|
||||
raw: &[calamine::Data],
|
||||
cfg: &DatasetConfig,
|
||||
patterns: &CompiledPatterns,
|
||||
) -> ParsedRow {
|
||||
let get = |idx: usize| -> String {
|
||||
raw.get(idx)
|
||||
.map(|cell| cell.to_string().trim().to_owned())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
let ho_ten = get(cfg.columns.ho_ten);
|
||||
let ngay_sinh = get(cfg.columns.ngay_sinh);
|
||||
let so_bao_danh = get(cfg.columns.so_bao_danh);
|
||||
let diem_thi = raw
|
||||
.get(cfg.columns.diem_thi)
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let ho_ten_ascii = to_ascii(&ho_ten);
|
||||
let scores = parse_scores(&diem_thi, patterns);
|
||||
let ngay_sinh_opt = if ngay_sinh.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ngay_sinh)
|
||||
};
|
||||
|
||||
ParsedRow {
|
||||
so_bao_danh,
|
||||
ho_ten,
|
||||
ho_ten_ascii,
|
||||
ngay_sinh: ngay_sinh_opt,
|
||||
scores,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests — 20 cases for to_ascii (real Vietnamese names)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Helper: assert to_ascii(input) == expected
|
||||
fn check(input: &str, expected: &str) {
|
||||
assert_eq!(
|
||||
to_ascii(input),
|
||||
expected,
|
||||
"to_ascii({input:?}) expected {expected:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_plain_latin() {
|
||||
check("Nguyen Van A", "nguyen van a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_hoa() {
|
||||
check("Nguyễn Thị Hoa", "nguyen thi hoa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_van_duc() {
|
||||
// đ/Đ replacement
|
||||
check("Trần Văn Đức", "tran van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_thi_my_duyen() {
|
||||
check("Lê Thị Mỹ Duyên", "le thi my duyen");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_pham_thi_lan() {
|
||||
check("Phạm Thị Lan", "pham thi lan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_bui_thi_thu() {
|
||||
check("Bùi Thị Thu", "bui thi thu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_hoang_van_truong() {
|
||||
check("Hoàng Văn Trường", "hoang van truong");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_do_thi_ngan() {
|
||||
// Đ uppercase at start
|
||||
check("Đỗ Thị Ngân", "do thi ngan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_van_khanh() {
|
||||
check("Nguyễn Văn Khánh", "nguyen van khanh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_trinh_thi_bich_ngoc() {
|
||||
check("Trịnh Thị Bích Ngọc", "trinh thi bich ngoc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vu_thi_dieu() {
|
||||
// ề = e + combining grave + combining circumflex (after NFD)
|
||||
check("Vũ Thị Diệu", "vu thi dieu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_tuong_vi() {
|
||||
check("Nguyễn Thị Tường Vi", "nguyen thi tuong vi");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_lowercase_d_stroke() {
|
||||
// Lowercase đ → d
|
||||
check("đặng thị hằng", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_uppercase_d_stroke() {
|
||||
check("ĐẶNG THỊ HẰNG", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_mixed_case() {
|
||||
check("NGUYỄN VĂN AN", "nguyen van an");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_thi_kim_anh() {
|
||||
check("Trần Thị Kim Anh", "tran thi kim anh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_phuong_thao() {
|
||||
check("Nguyễn Thị Phương Thảo", "nguyen thi phuong thao");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_van_long() {
|
||||
check("Lê Văn Long", "le van long");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vo_thi_xuan_mai() {
|
||||
check("Võ Thị Xuân Mai", "vo thi xuan mai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_empty_string() {
|
||||
check("", "");
|
||||
}
|
||||
|
||||
// --- Score parsing tests ---
|
||||
|
||||
fn make_patterns() -> CompiledPatterns {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("toan".into(), r"Toán:\s*(\d+(?:\.\d+)?)".into());
|
||||
map.insert("ngu_van".into(), r"Ngữ văn:\s*(\d+(?:\.\d+)?)".into());
|
||||
map.insert("vat_ly".into(), r"Vật lí:\s*(\d+(?:\.\d+)?)".into());
|
||||
CompiledPatterns::new(&map).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_single() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 8.5";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&8.5));
|
||||
assert!(scores.get("ngu_van").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_multiple() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&7.25));
|
||||
assert_eq!(scores.get("ngu_van"), Some(&6.0));
|
||||
assert_eq!(scores.get("vat_ly"), Some(&9.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_empty_cell() {
|
||||
let p = make_patterns();
|
||||
let scores = parse_scores("", &p);
|
||||
assert!(scores.is_empty());
|
||||
}
|
||||
|
||||
// --- Validation tests ---
|
||||
|
||||
fn default_validation() -> ValidationCfg {
|
||||
ValidationCfg {
|
||||
require_numeric_sbd: false,
|
||||
require_nonempty_name: true,
|
||||
require_nonempty_sbd: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_ok() {
|
||||
let v = default_validation();
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_sbd() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_name() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("", "12345678", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_non_numeric_sbd_rejected() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "12AB5678", &v, false, false),
|
||||
Err(SkipReason::NonNumericSbd)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_numeric_sbd_accepted() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_blank_row_skipped() {
|
||||
let v = default_validation();
|
||||
// strip_blank_rows=true AND all_blank=true → BlankRow
|
||||
assert_eq!(
|
||||
validate_row("", "", &v, true, true),
|
||||
Err(SkipReason::BlankRow)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
/// SQLite writer: DDL setup, batched INSERT OR REPLACE, VACUUM, stats output.
|
||||
///
|
||||
/// Mirrors build-lib.js createDb + the transaction loop in each build-database*.js.
|
||||
/// Stats output lines match the JS stdout exactly so existing CI log-greps still work.
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params_from_iter, Connection, ToSql};
|
||||
|
||||
use crate::config::DatasetConfig;
|
||||
use crate::error::BuildError;
|
||||
use crate::transform::ParsedRow;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB initialisation — mirrors build-lib.js createDb (delete + recreate)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Open (or recreate) the output SQLite database, execute the DDL from config,
|
||||
/// and return the open connection ready for inserts.
|
||||
pub fn open_db(db_path: &Path, cfg: &DatasetConfig) -> Result<Connection, BuildError> {
|
||||
// Mirror Node behaviour: delete existing file before creating (build-lib.js:54)
|
||||
if db_path.exists() {
|
||||
fs::remove_file(db_path).map_err(|e| BuildError::Io {
|
||||
path: db_path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if let Some(parent) = db_path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent).map_err(|e| BuildError::Io {
|
||||
path: parent.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
let conn = Connection::open(db_path)?;
|
||||
conn.execute_batch(&cfg.schema.ddl)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Ordered score field list — canonical INSERT column order from build-lib.js
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Fixed subject column order matching the INSERT statement in every config.
|
||||
/// NULL is bound for any subject not present in a given row's score map.
|
||||
pub const SCORE_FIELDS: &[&str] = &[
|
||||
"toan",
|
||||
"ngu_van",
|
||||
"vat_ly",
|
||||
"hoa_hoc",
|
||||
"sinh_hoc",
|
||||
"khtn",
|
||||
"lich_su",
|
||||
"dia_ly",
|
||||
"gdcd",
|
||||
"khxh",
|
||||
"tieng_anh",
|
||||
"tieng_phap",
|
||||
"tieng_nga",
|
||||
"tieng_trung",
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Insert a single parsed row inside an active transaction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Bind all fields from `row` into the prepared statement and execute it.
|
||||
/// `score_fields` should be the ordered list of subject columns the INSERT expects.
|
||||
pub fn insert_row(
|
||||
conn: &Connection,
|
||||
sql: &str,
|
||||
row: &ParsedRow,
|
||||
score_fields: &[&str],
|
||||
) -> Result<(), BuildError> {
|
||||
// Build positional params: so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, <scores...>
|
||||
let mut params: Vec<Box<dyn ToSql>> = Vec::with_capacity(4 + score_fields.len());
|
||||
params.push(Box::new(row.so_bao_danh.clone()));
|
||||
params.push(Box::new(row.ho_ten.clone()));
|
||||
params.push(Box::new(row.ho_ten_ascii.clone()));
|
||||
params.push(Box::new(row.ngay_sinh.clone()));
|
||||
|
||||
for field in score_fields {
|
||||
let val: Option<f64> = row.scores.get(*field).copied();
|
||||
params.push(Box::new(val));
|
||||
}
|
||||
|
||||
conn.execute(sql, params_from_iter(params.iter().map(|p| p.as_ref())))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Post-build: VACUUM + stats output
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run VACUUM and print statistics lines that mirror the Node scripts' stdout.
|
||||
/// The exact prefix tokens ("Source data rows", "DB rows", "Size:") are preserved
|
||||
/// so any log-grep in the deploy pipeline keeps working.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn finish_db(
|
||||
conn: &Connection,
|
||||
db_path: &Path,
|
||||
source_rows: u64,
|
||||
skipped: u64,
|
||||
errors: u64,
|
||||
dataset_label: &str, // e.g. "data/" or "data-old2/"
|
||||
_file_count: usize,
|
||||
is_old2: bool, // data-old2 uses different label for the skipped line
|
||||
) -> Result<(), BuildError> {
|
||||
conn.execute_batch("VACUUM")?;
|
||||
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let insertable = source_rows - skipped;
|
||||
|
||||
// Mirror exact JS stdout format for each dataset variant
|
||||
println!();
|
||||
if is_old2 {
|
||||
println!("Source non-blank data rows: {source_rows}");
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!("Source data rows (post-header): {source_rows}");
|
||||
if dataset_label.contains("old") {
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!(" skipped (empty/invalid): {skipped}");
|
||||
}
|
||||
}
|
||||
println!(" insertable: {insertable}");
|
||||
println!(" insert errors: {errors}");
|
||||
println!("DB rows (distinct SBD): {db_count}");
|
||||
|
||||
// Audit gap comment (mirrors build-database.js:80-83 for data/ only)
|
||||
if !dataset_label.contains("old") && errors == 0 {
|
||||
let gap = insertable as i64 - db_count;
|
||||
if gap == 0 {
|
||||
println!("Audit: OK — every source row made it in.");
|
||||
} else {
|
||||
println!("Audit: {gap} row(s) collapsed (duplicate SBDs overwriting).");
|
||||
}
|
||||
}
|
||||
|
||||
let sz = fs::metadata(db_path).map(|m| m.len()).unwrap_or(0);
|
||||
println!("Size: {:.1} MB", sz as f64 / 1024.0 / 1024.0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
# Test Fixtures
|
||||
|
||||
Anonymised `.xlsx` files for integration testing. All student PII has been replaced:
|
||||
|
||||
- `ho_ten` replaced with `Nguyen Van Test NNN` / `Tran Thi Test NNN` patterns
|
||||
- `so_bao_danh` replaced with sequential synthetic numbers (e.g. `10000001`)
|
||||
- `ngay_sinh` replaced with fixed synthetic dates
|
||||
- Scores are realistic random values in the 0–10 range
|
||||
|
||||
Files:
|
||||
|
||||
- `province-100.xlsx` — 100-row single-sheet file (simulates a normal province)
|
||||
- `hcm-overflow.xlsx` — 2-sheet file (200 rows Sheet1 + 200 rows Sheet2, simulating HCM overflow)
|
||||
- `province-numeric-sbd.xlsx` — 100 rows with strictly numeric SBDs (for data-old variant)
|
||||
|
||||
These files are generated by `tests/golden.rs` `generate_fixtures()` if they do not already exist
|
||||
on disk. The generator is pure Rust (uses the `zip` crate already pulled in via calamine).
|
||||
No external Python or Node tooling required for unit/integration tests.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,690 +0,0 @@
|
||||
/// Stage 5 golden tests — integration tests using anonymised fixture files.
|
||||
///
|
||||
/// Fixture files are generated in-process via raw OOXML + zip if they do not
|
||||
/// already exist on disk. No external Python or Node tooling required for the
|
||||
/// Rust-side tests. The Node golden comparison is marked #[ignore] when pnpm
|
||||
/// is not in PATH.
|
||||
use std::io::Write as IoWrite;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal OOXML xlsx generator
|
||||
//
|
||||
// Produces a valid .xlsx that calamine can read. Only uses the `zip` crate
|
||||
// which is already pulled in as a transitive dependency of calamine.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One row of cell data for a fixture sheet.
|
||||
struct XlsxRow {
|
||||
values: Vec<String>,
|
||||
}
|
||||
|
||||
/// Write a minimal .xlsx to `path` with the given sheets.
|
||||
/// `sheets`: Vec<(sheet_name, rows)> where rows[0] is the header.
|
||||
fn write_xlsx(path: &Path, sheets: &[(String, Vec<XlsxRow>)]) {
|
||||
use zip::{write::SimpleFileOptions, ZipWriter};
|
||||
|
||||
let file = std::fs::File::create(path).expect("create fixture xlsx");
|
||||
let mut zip = ZipWriter::new(file);
|
||||
let opts = SimpleFileOptions::default();
|
||||
|
||||
// [Content_Types].xml
|
||||
let mut content_types = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
content_types.push_str(&format!(
|
||||
r#" <Override PartName="/xl/worksheets/sheet{}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
"#,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
content_types.push_str("</Types>");
|
||||
zip.start_file("[Content_Types].xml", opts).unwrap();
|
||||
zip.write_all(content_types.as_bytes()).unwrap();
|
||||
|
||||
// _rels/.rels
|
||||
zip.start_file("_rels/.rels", opts).unwrap();
|
||||
zip.write_all(
|
||||
br#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// xl/_rels/workbook.xml.rels
|
||||
let mut wb_rels = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
wb_rels.push_str(&format!(
|
||||
r#" <Relationship Id="rId{}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{}.xml"/>
|
||||
"#,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb_rels.push_str("</Relationships>");
|
||||
zip.start_file("xl/_rels/workbook.xml.rels", opts).unwrap();
|
||||
zip.write_all(wb_rels.as_bytes()).unwrap();
|
||||
|
||||
// xl/workbook.xml
|
||||
let mut wb = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets>
|
||||
"#,
|
||||
);
|
||||
for (i, (name, _)) in sheets.iter().enumerate() {
|
||||
let escaped = xml_escape(name);
|
||||
wb.push_str(&format!(
|
||||
r#" <sheet name="{}" sheetId="{}" r:id="rId{}"/>
|
||||
"#,
|
||||
escaped,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb.push_str(" </sheets>\n</workbook>");
|
||||
zip.start_file("xl/workbook.xml", opts).unwrap();
|
||||
zip.write_all(wb.as_bytes()).unwrap();
|
||||
|
||||
// xl/worksheets/sheetN.xml
|
||||
for (i, (_, rows)) in sheets.iter().enumerate() {
|
||||
let mut ws = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
"#,
|
||||
);
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
ws.push_str(&format!(
|
||||
r#" <row r="{}">
|
||||
"#,
|
||||
row_idx + 1
|
||||
));
|
||||
for (col_idx, val) in row.values.iter().enumerate() {
|
||||
let col_letter = col_letter(col_idx);
|
||||
let cell_ref = format!("{}{}", col_letter, row_idx + 1);
|
||||
let escaped = xml_escape(val);
|
||||
ws.push_str(&format!(
|
||||
r#" <c r="{}" t="inlineStr"><is><t>{}</t></is></c>
|
||||
"#,
|
||||
cell_ref, escaped
|
||||
));
|
||||
}
|
||||
ws.push_str(" </row>\n");
|
||||
}
|
||||
ws.push_str(" </sheetData>\n</worksheet>");
|
||||
zip.start_file(&format!("xl/worksheets/sheet{}.xml", i + 1), opts)
|
||||
.unwrap();
|
||||
zip.write_all(ws.as_bytes()).unwrap();
|
||||
}
|
||||
|
||||
zip.finish().unwrap();
|
||||
}
|
||||
|
||||
fn col_letter(idx: usize) -> &'static str {
|
||||
const LETTERS: &[&str] = &[
|
||||
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R",
|
||||
"S", "T", "U", "V", "W", "X", "Y", "Z",
|
||||
];
|
||||
LETTERS[idx % 26]
|
||||
}
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture data builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn header_row() -> XlsxRow {
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"HO_TEN".into(),
|
||||
"NGAY_SINH".into(),
|
||||
"SO_BAO_DANH".into(),
|
||||
"DIEM_THI".into(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn data_row(idx: usize, scores: &str) -> XlsxRow {
|
||||
// Anonymised: name uses sequential pattern, SBD is purely synthetic
|
||||
let name = if idx % 2 == 0 {
|
||||
format!("Nguyen Van Test {:03}", idx)
|
||||
} else {
|
||||
format!("Tran Thi Test {:03}", idx)
|
||||
};
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
name,
|
||||
format!("15/0{}/{}", (idx % 9) + 1, 1999 + (idx % 5)),
|
||||
format!("1000{:04}", idx),
|
||||
scores.to_owned(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_scores(idx: usize) -> String {
|
||||
// Realistic scores in 0–10 range, varies by idx
|
||||
let toan = 4.0 + (idx % 60) as f64 / 10.0;
|
||||
let van = 3.5 + (idx % 65) as f64 / 10.0;
|
||||
format!("Toán: {toan:.1} Ngữ văn: {van:.1} Tiếng Anh: 7.5")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture file paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
// tests/fixtures/ relative to the crate root
|
||||
let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
p.push("tests");
|
||||
p.push("fixtures");
|
||||
p
|
||||
}
|
||||
|
||||
fn province_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-100.xlsx")
|
||||
}
|
||||
|
||||
fn hcm_overflow_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("hcm-overflow.xlsx")
|
||||
}
|
||||
|
||||
fn numeric_sbd_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-numeric-sbd.xlsx")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture generation — called once per test run if files missing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn ensure_fixtures() {
|
||||
let dir = fixtures_dir();
|
||||
std::fs::create_dir_all(&dir).expect("create fixtures dir");
|
||||
|
||||
// province-100.xlsx — 100 data rows, single sheet, with header
|
||||
if !province_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(&province_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
|
||||
// hcm-overflow.xlsx — 2 sheets × 200 rows each (no header on sheet 2)
|
||||
if !hcm_overflow_fixture_path().exists() {
|
||||
let mut sheet1 = vec![header_row()];
|
||||
for i in 0..200 {
|
||||
sheet1.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
// Sheet2: continuation rows, no header row (as in real HCM overflow)
|
||||
let mut sheet2 = Vec::new();
|
||||
for i in 200..400 {
|
||||
sheet2.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&hcm_overflow_fixture_path(),
|
||||
&[("Sheet1".to_owned(), sheet1), ("Sheet2".to_owned(), sheet2)],
|
||||
);
|
||||
}
|
||||
|
||||
// province-numeric-sbd.xlsx — strictly numeric SBDs for data-old config
|
||||
if !numeric_sbd_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Nguyen Van Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
format!("{:08}", 20000000 + i), // pure digits
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
write_xlsx(&numeric_sbd_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_data_config() -> xlsxread::config::DatasetConfig {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join("thptqg2017-data.toml");
|
||||
xlsxread::config::load_config(&cfg_path).expect("load data config")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Integration tests — pure Rust, no Node dependency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn province_100_builds_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(count, 100, "expected 100 rows from province-100 fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hcm_overflow_builds_400_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 400,
|
||||
"expected 400 rows (200 × 2 sheets) from hcm-overflow fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_old_first_sheet_only_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
// Use the overflow file but with data-old config (first sheet only → 200 rows)
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml");
|
||||
|
||||
// data-old: sheet_mode=first → only 200 rows from sheet1; but SBDs "1000NNNN" are
|
||||
// all digits so all pass the numeric guard
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 200,
|
||||
"data-old config should read only first sheet (200 rows)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_sbd_guard_rejects_non_numeric() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write a fixture with one non-numeric SBD mixed in
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
if i == 5 {
|
||||
"ABC123".to_owned()
|
||||
} else {
|
||||
format!("{:08}", 20000000 + i)
|
||||
},
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
let mixed_path = fixture_dir.join("mixed.xlsx");
|
||||
write_xlsx(&mixed_path, &[("Sheet1".to_owned(), rows)]);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml");
|
||||
|
||||
// Row i=5 has non-numeric SBD → rejected by data-old config
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 9,
|
||||
"non-numeric SBD row should be skipped by data-old config"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scores_parsed_correctly_into_db() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyen Van Test 001".to_owned(),
|
||||
"01/01/2000".to_owned(),
|
||||
"10000001".to_owned(),
|
||||
"Toán: 8.5 Ngữ văn: 7.0 Tiếng Anh: 9.25".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let (toan, van, anh): (f64, f64, f64) = conn
|
||||
.query_row(
|
||||
"SELECT toan, ngu_van, tieng_anh FROM student WHERE so_bao_danh = '10000001'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert!((toan - 8.5).abs() < 1e-9);
|
||||
assert!((van - 7.0).abs() < 1e-9);
|
||||
assert!((anh - 9.25).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_ascii_stored_correctly() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyễn Văn Đức".to_owned(),
|
||||
"".to_owned(),
|
||||
"20000001".to_owned(),
|
||||
"".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let ascii: String = conn
|
||||
.query_row(
|
||||
"SELECT ho_ten_ascii FROM student WHERE so_bao_danh = '20000001'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert_eq!(ascii, "nguyen van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_matches_after_build() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
// audit should match (100 distinct SBDs in xlsx == 100 rows in DB)
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(result.matched, "audit should match after build");
|
||||
assert_eq!(result.distinct_sbds, 100);
|
||||
assert_eq!(result.db_count, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_mismatch_detected() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write 10 rows to xlsx but build DB from only 5 rows
|
||||
let mut all_rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
all_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&fixture_dir.join("all.xlsx"),
|
||||
&[("Sheet1".to_owned(), all_rows)],
|
||||
);
|
||||
|
||||
// Build DB with only first 5 rows in a different file
|
||||
let build_dir = dir.join("build_input");
|
||||
std::fs::create_dir_all(&build_dir).unwrap();
|
||||
let mut five_rows = vec![header_row()];
|
||||
for i in 0..5 {
|
||||
five_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&build_dir.join("five.xlsx"),
|
||||
&[("Sheet1".to_owned(), five_rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&build_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
// audit against fixture_dir (10 xlsx rows) but DB has 5 rows → mismatch
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(!result.matched, "audit should not match (10 xlsx vs 5 db)");
|
||||
assert_eq!(result.distinct_sbds, 10);
|
||||
assert_eq!(result.db_count, 5);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Golden test: compare Rust DB vs Node DB on identical fixture
|
||||
// Marked #[ignore] when pnpm / node is not in PATH — CI installs them first.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn golden_rust_matches_node_db() {
|
||||
// This test requires: pnpm, node, and the thptqg2017 package to be installed
|
||||
// Run with: cargo test -- --ignored golden_rust_matches_node_db
|
||||
let which_pnpm = std::process::Command::new("which")
|
||||
.arg("pnpm")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false);
|
||||
if !which_pnpm {
|
||||
eprintln!("pnpm not in PATH — skipping golden test");
|
||||
return;
|
||||
}
|
||||
|
||||
let dir = tempdir();
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
// Build with Rust
|
||||
let rust_db = dir.join("rust.db");
|
||||
run_build_cmd(&fixture_dir, &rust_db, "thptqg2017-data.toml");
|
||||
|
||||
// Build with Node (run build-database.js with DATA_DIR / DB_PATH overrides)
|
||||
// Node script reads env-vars via a thin wrapper — see scripts/build-database.js
|
||||
// For now: diff via SELECT * ORDER BY so_bao_danh
|
||||
let node_db = dir.join("node.db");
|
||||
let status = std::process::Command::new("pnpm")
|
||||
.args(["exec", "node", "scripts/build-database.js"])
|
||||
.env("OVERRIDE_SRC_DIR", fixture_dir.to_str().unwrap())
|
||||
.env("OVERRIDE_DB_PATH", node_db.to_str().unwrap())
|
||||
.current_dir(
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap(),
|
||||
)
|
||||
.status()
|
||||
.expect("failed to run node build script");
|
||||
|
||||
if !status.success() {
|
||||
panic!("Node build script failed with: {status}");
|
||||
}
|
||||
|
||||
// Row-by-row comparison
|
||||
diff_dbs(&rust_db, &node_db);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn tempdir() -> PathBuf {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"xlsxread-test-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.subsec_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
base
|
||||
}
|
||||
|
||||
fn run_build_cmd(input_dir: &Path, db_path: &Path, config_name: &str) {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join(config_name);
|
||||
|
||||
let cfg = xlsxread::config::load_config(&cfg_path)
|
||||
.unwrap_or_else(|e| panic!("load config {config_name}: {e}"));
|
||||
let patterns =
|
||||
xlsxread::transform::CompiledPatterns::new(&cfg.scores).expect("compile patterns");
|
||||
|
||||
// Collect files
|
||||
let mut files: Vec<PathBuf> = std::fs::read_dir(input_dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let l = e.to_lowercase();
|
||||
l == "xls" || l == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let conn = xlsxread::writer::open_db(db_path, &cfg).expect("open db");
|
||||
conn.execute_batch("BEGIN").unwrap();
|
||||
|
||||
for file in &files {
|
||||
xlsxread::reader::process_file(file, &cfg, |_, raw| {
|
||||
let all_blank = xlsxread::reader::is_all_blank(raw);
|
||||
if cfg.reader.strip_blank_rows && all_blank {
|
||||
return;
|
||||
}
|
||||
let ho_ten = raw
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
if xlsxread::transform::validate_row(
|
||||
&ho_ten,
|
||||
&so_bao_danh,
|
||||
&cfg.validation,
|
||||
cfg.reader.strip_blank_rows,
|
||||
all_blank,
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let row = xlsxread::transform::transform_row(raw, &cfg, &patterns);
|
||||
let _ = xlsxread::writer::insert_row(
|
||||
&conn,
|
||||
&cfg.insert.sql,
|
||||
&row,
|
||||
xlsxread::writer::SCORE_FIELDS,
|
||||
);
|
||||
})
|
||||
.expect("process file");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT").unwrap();
|
||||
conn.execute_batch("VACUUM").unwrap();
|
||||
}
|
||||
|
||||
fn query_count(db_path: &Path) -> i64 {
|
||||
let conn = rusqlite::Connection::open(db_path).expect("open db for count");
|
||||
conn.query_row("SELECT COUNT(*) FROM student", [], |r| r.get(0))
|
||||
.expect("count query")
|
||||
}
|
||||
|
||||
fn diff_dbs(a: &Path, b: &Path) {
|
||||
let conn_a = rusqlite::Connection::open(a).unwrap();
|
||||
|
||||
// Attach b as "other"
|
||||
conn_a
|
||||
.execute_batch(&format!("ATTACH DATABASE '{}' AS other", b.display()))
|
||||
.unwrap();
|
||||
|
||||
// Rows in a not in b
|
||||
let missing_in_b: i64 = conn_a
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM main.student s
|
||||
WHERE NOT EXISTS (SELECT 1 FROM other.student o WHERE o.so_bao_danh = s.so_bao_danh)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Rows in b not in a
|
||||
let missing_in_a: i64 = conn_a
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM other.student o
|
||||
WHERE NOT EXISTS (SELECT 1 FROM main.student s WHERE s.so_bao_danh = o.so_bao_danh)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
missing_in_b, 0,
|
||||
"{missing_in_b} rows in Rust DB missing from Node DB"
|
||||
);
|
||||
assert_eq!(
|
||||
missing_in_a, 0,
|
||||
"{missing_in_a} rows in Node DB missing from Rust DB"
|
||||
);
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user