diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 71dfc46..a8ae901 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -3,6 +3,11 @@ name: Deploy to GitHub Pages on: push: branches: [main] + # Pull requests run the build job only. Before this, the workflow triggered on + # main-push and workflow_dispatch alone, so "verify on a branch first" was not + # actually possible: pushing to a branch ran nothing, and dispatching from one + # published that branch straight to the live site. + pull_request: workflow_dispatch: permissions: @@ -17,42 +22,74 @@ concurrency: jobs: build: runs-on: ubuntu-latest + env: + # Every Go module here is cgo-free — grate, excelize, yaml.v3, x/net, + # x/text and modernc.org/sqlite — so no C toolchain is needed. Set + # explicitly rather than relying on the default. + CGO_ENABLED: '0' steps: - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-go@v5 with: - workspaces: parser + go-version: '1.26' + cache-dependency-path: | + parser/go.sum + crawler/go.sum + assembler/go.sum + # web/ is the only npm project in the repository; the other stages are Go. - uses: actions/setup-node@v4 with: node-version: '24' cache: 'npm' - cache-dependency-path: package-lock.json + cache-dependency-path: web/package-lock.json - - run: npm ci + - name: Install web dependencies + working-directory: web + run: npm ci - # One parser binary builds every dataset; build-db.js reads the dataset - # list from src/datasets.js and gzips each database in place, leaving no - # uncompressed file behind. - - name: Build databases + # The reader-fidelity suite compares every real input file against a + # committed hash oracle, so it is the regression guard for the whole + # reader. Runs before anything is built. + - name: Test parser + run: go -C parser test ./... + + # The crawler is not part of the build — it only refreshes data/ by hand. + # It is still tested here so it cannot rot unnoticed, and because its + # fixture test guards the parser: input filenames decide which row + # survives a duplicate exam number. + - name: Test crawler + run: go -C crawler test ./... + + - name: Lint web + working-directory: web + run: npm run lint + + # excelize carries an open advisory, and the 2017 refresh runbook feeds + # network-downloaded spreadsheets straight into the parser. + - name: Vulnerability scan run: | - npm run build:rust - npm run build:db + go install golang.org/x/vuln/cmd/govulncheck@latest + GOVULNCHECK="$(go env GOPATH)/bin/govulncheck" + for m in parser crawler assembler; do (cd "$m" && "$GOVULNCHECK" ./...); done - # One Vite build produces every page. scripts/assemble-site.js copies the - # emitted index.html to each dataset path (and the legacy nested URLs), - # then fails the job if any uncompressed database reached the artifact. - - name: Build and assemble site - run: npm run build:site + # One command runs the whole pipeline: compile the parser, build and + # verify each database against its registry row count, compress it, build + # the web app, and assemble _site — refusing to continue if a database is + # short, an artifact looks truncated, or one is missing entirely. + - name: Build site + run: go -C assembler run ./cmd/assemble - uses: actions/upload-pages-artifact@v3 with: path: _site deploy: + # Guarded to main. Without this, a workflow_dispatch from any branch would + # publish that branch's output to the live site, and concurrency + # cancel-in-progress would kill an in-flight good deploy on the way. + if: github.ref == 'refs/heads/main' needs: build runs-on: ubuntu-latest environment: diff --git a/.gitignore b/.gitignore index c67537c..35f1649 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,9 @@ dist/ ### Generated databases and Vite publicDir staging ### .build/ -### Rust build artefacts ### -parser/target/ - ### Assembled Pages artifact ### _site/ + +# Parser build output and regenerable ground-truth dumps +parser/bin/ +parser/testdata/dumps/ diff --git a/README.md b/README.md index da6a3df..627f52a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Tra cứu điểm thi THPT Quốc gia — exam-score lookup for Vietnam's national high school graduation exam. Client-side SQL (sql.js) over a SQLite database built -from the ministry's raw `.xls` score files by the Rust `xlsxread` parser. +from the ministry's raw `.xls` score files by the Go `xlsxread` parser. Live at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**. @@ -10,55 +10,84 @@ Live at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**. | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | [/2016/](https://tiennm99.github.io/thptqg/2016/) | | `2017` | 2017 | 861,068 | [/2017/](https://tiennm99.github.io/thptqg/2017/) | -| `2017-old` | 2017 | 847,348 | [/2017-old/](https://tiennm99.github.io/thptqg/2017-old/) | -| `2017-old2` | 2017 | 679,764 | [/2017-old2/](https://tiennm99.github.io/thptqg/2017-old2/) | -The three 2017 datasets are successive publications of the same exam and they -disagree; all three are kept so the differences stay inspectable. +Two earlier 2017 publications (`2017-old`, `2017-old2`) were kept for a while +because they disagreed with the current one. They have been removed; they remain +in git history. ## Layout +The repository is one directory per pipeline stage, plus the two stores they +pass between them. + ``` -index.html + src/ the frontend — one app serving all four datasets and the hub - datasets.js the four dataset ids and their per-dataset content - router.js pathname → dataset -data// raw Excel files, one directory per dataset -parser/ the Rust parser - src/schema.rs canonical 22-column table: DDL, INSERT, subject regexes - configs/.toml per-dataset parse rules only, no SQL - scripts/ database build, crawler, parity verification -scripts/ site assembly -docs/ architecture, data pipeline, deployment +crawler/ Go — re-fetches the source spreadsheets → data/ +parser/ Go — Excel to SQLite data/ → .db +assembler/ Go — verifies, compresses, builds, assembles .db + web/ → _site/ +web/ npm — the frontend, one Vite app for every dataset +data// raw Excel files, one directory per dataset +datasets.json the registry: which datasets exist, and their expected size +docs/ architecture, data pipeline, deployment ``` +Each stage runs on its own and hands its output to the next through the stores. +`web/` is the only npm project; the three stages are independent Go modules. + +`datasets.json` is the contract between them. It is JSON because Go and the Vite +app both read it and neither needs a dependency to do so; presentation stays in +`web/src/datasets.js`, keyed by id, which fails loudly if the two disagree. + The dataset id is one identifier end to end: ``` -data/2017-old/ → parser/configs/2017-old.toml → db/2017-old.db.gz → /thptqg/2017-old/ +data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ ``` ## Build ```bash -npm ci -npm run build:rust # compile the parser -npm run build:db # build + gzip all four databases (add an id for just one) -npm run build:site # one Vite build, then assemble into _site/ +(cd web && npm ci) +go -C assembler run ./cmd/assemble # databases, then the site, into _site/ npx serve _site ``` +That one command compiles the parser, builds and verifies each database against +its registry row count, compresses it, builds the web app and assembles `_site` — +refusing to continue if a database is short, an artifact looks truncated, or one +is missing altogether. Sub-steps when iterating: + +```bash +go -C assembler run ./cmd/assemble db 2017 # one database +go -C assembler run ./cmd/assemble site # web build and _site only +(cd web && npm run dev) # the app against staged databases +``` + +The source spreadsheets are committed, so a crawl is only needed to refresh +them: + +```bash +go -C crawler run ./cmd/crawl 2016 +go -C crawler run ./cmd/crawl 2017 +``` + +Each reads the download links out of the article that published the dataset, so +no link list is kept in the repository. Crawling is idempotent — files already +present are skipped — and is never part of the build. + Pushing to `main` runs the same steps in `.github/workflows/deploy-pages.yml` and publishes to GitHub Pages. ## Adding a dataset 1. Put the Excel files in `data//` -2. Add `parser/configs/.toml` — sheet mode, column indices, validation +2. Add `parser/configs/.yml` — sheet mode, column indices, validation guards. No SQL; the schema is canonical. -3. Add an entry to `DATASETS` in `src/datasets.js` +3. Add an entry to `datasets.json` with its expected row count and size +4. Add the matching presentation to `CONTENT` in `web/src/datasets.js` -Everything else follows: the build script, the site assembly and the router all -read that one list, and the UI adapts to whichever columns the dataset fills. +Everything else follows: the assembler, the router and the hub all read the +registry, and the UI adapts to whichever columns the dataset fills. Steps 3 and 4 +check each other, so forgetting either one fails rather than half-working. ## Docs diff --git a/assembler/cmd/assemble/main.go b/assembler/cmd/assemble/main.go new file mode 100644 index 0000000..adaa95b --- /dev/null +++ b/assembler/cmd/assemble/main.go @@ -0,0 +1,129 @@ +// Command assemble turns source data and the web app into the directory +// GitHub Pages publishes. +// +// assemble # databases, then the site +// assemble db # databases only (add ids to limit: assemble db 2017) +// assemble site # web build and _site only, reusing staged databases +// +// It sequences the other stages rather than doing their work: the parser reads +// spreadsheets, Vite bundles the app, and this decides what runs, checks what +// came out, and refuses to publish anything that looks wrong. +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/tiennm99/thptqg/assembler/internal/databases" + "github.com/tiennm99/thptqg/assembler/internal/registry" + "github.com/tiennm99/thptqg/assembler/internal/site" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "assemble: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, `usage: assemble [step] [dataset...] + +Steps: + (none) databases, then the site + db build, verify and compress the databases + site build the web app and assemble _site + +Naming datasets limits the db step to those; the site step always covers all of +them, since a partial site would publish links to databases it did not build. +`) +} + +func run(args []string) error { + step := "" + if len(args) > 0 { + switch args[0] { + case "db", "site": + step, args = args[0], args[1:] + case "-h", "--help": + usage() + return nil + default: + // Bare dataset ids are a natural thing to type; treat them as the + // db step rather than rejecting them. + step = "db" + } + } + + root, err := repoRoot() + if err != nil { + return err + } + + all, err := registry.Load(root) + if err != nil { + return err + } + + if step == "" || step == "db" { + selected, err := registry.Select(all, args) + if err != nil { + return err + } + if err := buildDatabases(root, all, selected); err != nil { + return err + } + } + + if step == "" || step == "site" { + sp := site.DefaultPaths(root) + if err := site.BuildWeb(sp); err != nil { + return err + } + if err := site.Assemble(sp, all); err != nil { + return err + } + } + return nil +} + +func buildDatabases(root string, all, selected []registry.Dataset) error { + p := databases.DefaultPaths(root) + + // Sweep first: a dataset dropped from the registry leaves its .db.gz behind, + // and the site assembly copies the staging directory wholesale. + if err := databases.Clean(p, all); err != nil { + return err + } + + bin, err := databases.BuildParser(p) + if err != nil { + return err + } + for _, d := range selected { + if err := databases.Build(p, bin, d); err != nil { + return err + } + } + return nil +} + +// repoRoot walks up from the working directory to the directory holding +// datasets.json, so the command works from anywhere in the tree. +func repoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "datasets.json")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("no datasets.json found in any parent of the working directory") + } + dir = parent + } +} diff --git a/assembler/go.mod b/assembler/go.mod new file mode 100644 index 0000000..481fa27 --- /dev/null +++ b/assembler/go.mod @@ -0,0 +1,17 @@ +module github.com/tiennm99/thptqg/assembler + +go 1.26.5 + +require modernc.org/sqlite v1.56.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/assembler/go.sum b/assembler/go.sum new file mode 100644 index 0000000..1932692 --- /dev/null +++ b/assembler/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/assembler/internal/databases/databases.go b/assembler/internal/databases/databases.go new file mode 100644 index 0000000..0bf077f --- /dev/null +++ b/assembler/internal/databases/databases.go @@ -0,0 +1,217 @@ +// Package databases builds, verifies and compresses one SQLite file per +// dataset. +// +// VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra. +// +// Nothing between the parser and the published site otherwise asserts that a +// database has data in it. The parser logs a file-level failure and continues, +// returns success regardless, and finishes cleanly even at zero rows; the site +// assembly only inspects filenames. So a reader that silently under-produced +// would publish a truncated dataset with green CI and no red signal anywhere. +// +// The guards below close that: a build whose row count does not match the +// registry, or whose artifact is implausibly small, fails the pipeline. +package databases + +import ( + "compress/gzip" + "database/sql" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + _ "modernc.org/sqlite" // pure-Go driver: the pipeline stays cgo-free + + "github.com/tiennm99/thptqg/assembler/internal/registry" +) + +// driverName is modernc.org/sqlite's registered name. +const driverName = "sqlite" + +// minSizeRatio: a gzipped database far below its usual size means a truncated +// build, even if the row count somehow passed. +const minSizeRatio = 0.9 + +// Paths locates the pieces this package needs. +type Paths struct { + // Root is the repository root. + Root string + // Parser is the parser module directory. + Parser string + // OutDir is where the databases are staged — the directory Vite publishes. + OutDir string +} + +// DefaultPaths derives the standard layout from the repository root. +func DefaultPaths(root string) Paths { + return Paths{ + Root: root, + Parser: filepath.Join(root, "parser"), + OutDir: filepath.Join(root, ".build", "public", "db"), + } +} + +// BuildParser compiles the parser binary and returns its path. +// +// Compiling here rather than expecting a prebuilt binary keeps the pipeline one +// command. Go caches the work, so repeat runs cost almost nothing. +func BuildParser(p Paths) (string, error) { + bin := filepath.Join(p.Parser, "bin", "xlsxread") + cmd := exec.Command("go", "-C", p.Parser, "build", "-o", "bin/xlsxread", "./cmd/xlsxread") + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("compiling the parser: %w", err) + } + return bin, nil +} + +// Build runs the parser for one dataset, verifies the result and compresses it. +// +// Only the .gz survives: shipping a 100+ MB uncompressed database is made +// structurally impossible rather than left to a cleanup step. +func Build(p Paths, bin string, d registry.Dataset) error { + if err := os.MkdirAll(p.OutDir, 0o755); err != nil { + return err + } + db := filepath.Join(p.OutDir, d.ID+".db") + + cmd := exec.Command(bin, + "build", + "--schema", filepath.Join(p.Parser, "configs", d.ID+".yml"), + "--input", filepath.Join(p.Root, "data", d.ID), + "--output", db, + ) + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s: parser failed: %w", d.ID, err) + } + + rows, err := countRows(db) + if err != nil { + return fmt.Errorf("%s: %w", d.ID, err) + } + if rows != d.ExpectedRows { + return fmt.Errorf( + "%s: row count %d, expected %d\nRefusing to publish — the build did not reproduce the known dataset", + d.ID, rows, d.ExpectedRows) + } + fmt.Printf(" ✓ %s: %d rows (matches expected)\n", d.ID, rows) + + gz, size, err := compress(db) + if err != nil { + return fmt.Errorf("%s: %w", d.ID, err) + } + + sizeMb := float64(size) / 1024 / 1024 + if min := d.DbSizeMb * minSizeRatio; sizeMb < min { + return fmt.Errorf( + "%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+ + "Refusing to publish — the artifact looks truncated", + d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb) + } + + fmt.Printf(" → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb) + return nil +} + +// countRows opens the database read-only and counts what was written. +func countRows(path string) (int64, error) { + conn, err := sql.Open(driverName, "file:"+path+"?mode=ro") + if err != nil { + return 0, err + } + defer conn.Close() + + var n int64 + if err := conn.QueryRow("SELECT COUNT(*) FROM student").Scan(&n); err != nil { + return 0, fmt.Errorf("counting rows: %w", err) + } + return n, nil +} + +// compress gzips path to path+".gz" and removes the original, returning the +// compressed path and its size. +// +// The source is deleted only after the compressed file is closed successfully, +// so a failure part-way through leaves the database rather than losing it. +func compress(path string) (string, int64, error) { + in, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer in.Close() + + gzPath := path + ".gz" + out, err := os.Create(gzPath) + if err != nil { + return "", 0, err + } + + zw, err := gzip.NewWriterLevel(out, gzip.BestCompression) + if err != nil { + out.Close() + return "", 0, err + } + if _, err := io.Copy(zw, in); err != nil { + zw.Close() + out.Close() + os.Remove(gzPath) + return "", 0, err + } + if err := zw.Close(); err != nil { + out.Close() + os.Remove(gzPath) + return "", 0, err + } + if err := out.Close(); err != nil { + os.Remove(gzPath) + return "", 0, err + } + + if err := in.Close(); err != nil { + return "", 0, err + } + if err := os.Remove(path); err != nil { + return "", 0, fmt.Errorf("removing the uncompressed database: %w", err) + } + + st, err := os.Stat(gzPath) + if err != nil { + return "", 0, err + } + return gzPath, st.Size(), nil +} + +// Clean removes staged artifacts for datasets that are no longer in the +// registry. Without this a removed dataset's .db.gz lingers in the staging +// directory, and the site assembly copies that directory wholesale — so the +// dead database would be published again. +func Clean(p Paths, keep []registry.Dataset) error { + entries, err := os.ReadDir(p.OutDir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + wanted := make(map[string]bool, len(keep)*2) + for _, d := range keep { + wanted[d.ID+".db"] = true + wanted[d.ID+".db.gz"] = true + } + + for _, e := range entries { + if e.IsDir() || wanted[e.Name()] { + continue + } + full := filepath.Join(p.OutDir, e.Name()) + if err := os.Remove(full); err != nil { + return err + } + fmt.Printf(" removed stale artifact %s\n", e.Name()) + } + return nil +} diff --git a/assembler/internal/databases/databases_test.go b/assembler/internal/databases/databases_test.go new file mode 100644 index 0000000..05a9c27 --- /dev/null +++ b/assembler/internal/databases/databases_test.go @@ -0,0 +1,108 @@ +package databases + +import ( + "compress/gzip" + "io" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/tiennm99/thptqg/assembler/internal/registry" +) + +// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that +// shipping a 100+ MB uncompressed database is structurally impossible rather +// than left to a cleanup step. +func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "2016.db") + body := []byte("pretend this is a SQLite file") + if err := os.WriteFile(src, body, 0o644); err != nil { + t.Fatal(err) + } + + gzPath, size, err := compress(src) + if err != nil { + t.Fatal(err) + } + if gzPath != src+".gz" || size <= 0 { + t.Fatalf("gzPath=%q size=%d", gzPath, size) + } + if _, err := os.Stat(src); !os.IsNotExist(err) { + t.Error("the uncompressed database must not survive") + } + + f, err := os.Open(gzPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + zr, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(zr) + if err != nil { + t.Fatal(err) + } + if string(got) != string(body) { + t.Errorf("round-trip gave %q", got) + } +} + +func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{ + "2016.db.gz", "2017.db.gz", + "2017-old.db.gz", // dropped from the registry + "2017-old2.db.gz", // dropped from the registry + "2016.db-journal", // interrupted run + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + p := Paths{OutDir: dir} + keep := []registry.Dataset{{ID: "2016"}, {ID: "2017"}} + if err := Clean(p, keep); err != nil { + t.Fatal(err) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var left []string + for _, e := range entries { + left = append(left, e.Name()) + } + slices.Sort(left) + + want := []string{"2016.db.gz", "2017.db.gz"} + if !slices.Equal(left, want) { + t.Errorf("left %v, want %v", left, want) + } +} + +// TestCleanToleratesAnAbsentStagingDirectory: a fresh checkout has never built +// anything, and that is not an error. +func TestCleanToleratesAnAbsentStagingDirectory(t *testing.T) { + p := Paths{OutDir: filepath.Join(t.TempDir(), "never-created")} + if err := Clean(p, nil); err != nil { + t.Errorf("Clean on a missing directory should succeed, got %v", err) + } +} + +func TestDefaultPaths(t *testing.T) { + p := DefaultPaths("/repo") + if p.Parser != filepath.Join("/repo", "parser") { + t.Errorf("Parser = %q", p.Parser) + } + // The staging directory must be the one Vite publishes, or the databases + // never reach the site. + if p.OutDir != filepath.Join("/repo", ".build", "public", "db") { + t.Errorf("OutDir = %q", p.OutDir) + } +} diff --git a/assembler/internal/registry/registry.go b/assembler/internal/registry/registry.go new file mode 100644 index 0000000..7622185 --- /dev/null +++ b/assembler/internal/registry/registry.go @@ -0,0 +1,80 @@ +// Package registry reads the repository-root datasets.json. +// +// That file is the one place every stage agrees on what exists. The Vite app +// reads it too, which is why it is JSON: Go and the browser both parse it +// without a dependency. +package registry + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// Dataset is one entry in the registry. +type Dataset struct { + ID string `json:"id"` + // ExpectedRows is exact. The inputs are frozen historical exam results, so + // a deviation of even one row means something changed that nobody intended. + ExpectedRows int64 `json:"expectedRows"` + // DbSizeMb is the usual size of the gzipped database, used to catch a + // build that produced a plausible row count but a truncated artifact. + DbSizeMb float64 `json:"dbSizeMb"` +} + +type file struct { + Datasets []Dataset `json:"datasets"` +} + +// Load reads datasets.json from the repository root. +func Load(root string) ([]Dataset, error) { + path := filepath.Join(root, "datasets.json") + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("cannot read the dataset registry: %w", err) + } + + var f file + if err := json.Unmarshal(b, &f); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if len(f.Datasets) == 0 { + return nil, fmt.Errorf("%s declares no datasets", path) + } + + for _, d := range f.Datasets { + switch { + case d.ID == "": + return nil, fmt.Errorf("%s: a dataset has no id", path) + case d.ExpectedRows <= 0: + return nil, fmt.Errorf("%s: %s has no expectedRows; the build guard needs it", path, d.ID) + case d.DbSizeMb <= 0: + return nil, fmt.Errorf("%s: %s has no dbSizeMb; the size guard needs it", path, d.ID) + } + } + return f.Datasets, nil +} + +// Select returns the named datasets, or all of them when none are named. +func Select(all []Dataset, ids []string) ([]Dataset, error) { + if len(ids) == 0 { + return all, nil + } + byID := make(map[string]Dataset, len(all)) + known := make([]string, 0, len(all)) + for _, d := range all { + byID[d.ID] = d + known = append(known, d.ID) + } + + out := make([]Dataset, 0, len(ids)) + for _, id := range ids { + d, ok := byID[id] + if !ok { + return nil, fmt.Errorf("unknown dataset %q (known: %v)", id, known) + } + out = append(out, d) + } + return out, nil +} diff --git a/assembler/internal/registry/registry_test.go b/assembler/internal/registry/registry_test.go new file mode 100644 index 0000000..e1c8d06 --- /dev/null +++ b/assembler/internal/registry/registry_test.go @@ -0,0 +1,93 @@ +package registry + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeRegistry(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "datasets.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestLoadsTheRealRegistry(t *testing.T) { + root := filepath.Join("..", "..", "..") + got, err := Load(root) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 { + t.Fatal("no datasets") + } + for _, d := range got { + if d.ID == "" || d.ExpectedRows <= 0 || d.DbSizeMb <= 0 { + t.Errorf("incomplete entry: %+v", d) + } + // Every declared dataset must have somewhere to read from and a parse + // config, or the build fails much later with a worse message. + for _, p := range []string{ + filepath.Join(root, "data", d.ID), + filepath.Join(root, "parser", "configs", d.ID+".yml"), + } { + if _, err := os.Stat(p); err != nil { + t.Errorf("%s: missing %s", d.ID, p) + } + } + } +} + +// TestIncompleteEntriesAreRejected: a dataset missing either guard figure would +// otherwise publish unverified. Both are load-bearing, so neither may default. +func TestIncompleteEntriesAreRejected(t *testing.T) { + for name, body := range map[string]string{ + "no expectedRows": `{"datasets":[{"id":"x","dbSizeMb":1}]}`, + "no dbSizeMb": `{"datasets":[{"id":"x","expectedRows":1}]}`, + "no id": `{"datasets":[{"expectedRows":1,"dbSizeMb":1}]}`, + "zero rows": `{"datasets":[{"id":"x","expectedRows":0,"dbSizeMb":1}]}`, + "empty": `{"datasets":[]}`, + } { + t.Run(name, func(t *testing.T) { + if _, err := Load(writeRegistry(t, body)); err == nil { + t.Error("expected an error") + } + }) + } +} + +func TestMissingAndMalformedRegistry(t *testing.T) { + if _, err := Load(t.TempDir()); err == nil { + t.Error("a missing registry must fail") + } + if _, err := Load(writeRegistry(t, "{not json")); err == nil { + t.Error("a malformed registry must fail") + } +} + +func TestSelect(t *testing.T) { + all := []Dataset{{ID: "2016"}, {ID: "2017"}} + + got, err := Select(all, nil) + if err != nil || len(got) != 2 { + t.Errorf("no ids should select everything: %v %v", got, err) + } + + got, err = Select(all, []string{"2017"}) + if err != nil || len(got) != 1 || got[0].ID != "2017" { + t.Errorf("Select(2017) = %v, %v", got, err) + } + + // A typo must not silently build nothing. + _, err = Select(all, []string{"2018"}) + if err == nil { + t.Fatal("an unknown id must fail") + } + if !strings.Contains(err.Error(), "2018") || !strings.Contains(err.Error(), "2016") { + t.Errorf("the error should name the bad id and the known ones, got: %v", err) + } +} diff --git a/assembler/internal/site/site.go b/assembler/internal/site/site.go new file mode 100644 index 0000000..f799bb4 --- /dev/null +++ b/assembler/internal/site/site.go @@ -0,0 +1,210 @@ +// Package site turns the web app and the staged databases into the directory +// GitHub Pages publishes. +// +// The app resolves its dataset from the URL, so every page is the same +// index.html. Because Vite's `base` is absolute (/thptqg/), that file references +// /thptqg/assets/... no matter which directory it is served from — so copying it +// to each dataset path produces a real static file at every URL. +// +// GitHub Pages serves those as directory indexes, which is why this needs no +// SPA 404-fallback redirect. That matters beyond tidiness: the usual fallback +// rewrites the URL and would interfere with the ?q= deep links the app relies +// on. +package site + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/tiennm99/thptqg/assembler/internal/registry" +) + +// Paths locates the pieces this package needs. +type Paths struct { + Root string + // Web is the Vite project directory. + Web string + // Dist is where Vite emits, inside the web workspace. + Dist string + // Site is the artifact the deploy action uploads, at the repository root. + Site string +} + +// DefaultPaths derives the standard layout from the repository root. +func DefaultPaths(root string) Paths { + web := filepath.Join(root, "web") + return Paths{ + Root: root, + Web: web, + Dist: filepath.Join(web, "dist"), + Site: filepath.Join(root, "_site"), + } +} + +// BuildWeb runs the Vite build. +// +// Shelling out to npm is not a wart: Vite is a Node tool, and web/ is the only +// npm project left in the repository. This stage owns the sequencing, not the +// bundling. +func BuildWeb(p Paths) error { + cmd := exec.Command("npm", "run", "build") + cmd.Dir = p.Web + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("vite build: %w", err) + } + return nil +} + +// Assemble copies the build to one directory per dataset and checks the result. +func Assemble(p Paths, datasets []registry.Dataset) error { + index := filepath.Join(p.Dist, "index.html") + if _, err := os.Stat(index); err != nil { + return fmt.Errorf("no build found at %s — run the web build first", p.Dist) + } + + if err := os.RemoveAll(p.Site); err != nil { + return err + } + if err := os.MkdirAll(p.Site, 0o755); err != nil { + return err + } + + // Base build: index.html, assets/, and the gzipped databases from publicDir. + if err := copyTree(p.Dist, p.Site); err != nil { + return err + } + + // Unknown paths render the hub rather than the default Pages 404. + if err := copyFile(index, filepath.Join(p.Site, "404.html")); err != nil { + return err + } + + // One entry point per dataset. + for _, d := range datasets { + dir := filepath.Join(p.Site, d.ID) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + if err := copyFile(index, filepath.Join(dir, "index.html")); err != nil { + return err + } + } + + if err := checkDatabasesPresent(p.Site, datasets); err != nil { + return err + } + if err := checkNoRawDatabases(p.Site); err != nil { + return err + } + + fmt.Printf("assembled %s\n", p.Site) + fmt.Printf(" /thptqg/\n /thptqg/404.html\n") + for _, d := range datasets { + fmt.Printf(" /thptqg/%s\n", d.ID) + } + return nil +} + +// checkDatabasesPresent: every dataset must have shipped its database. +// +// Without this the site assembles happily with an empty db/ directory — every +// page renders, every query 404s, and CI stays green. That is the failure this +// catches; the size and row-count guards only run when a database was built at +// all. +func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error { + var missing []string + for _, d := range datasets { + gz := filepath.Join(siteDir, "db", d.ID+".db.gz") + st, err := os.Stat(gz) + if err != nil || st.Size() == 0 { + missing = append(missing, d.ID+".db.gz") + } + } + if len(missing) > 0 { + return fmt.Errorf( + "no database in the site output for: %s\n"+ + "Every page would render and every query would 404. Build the databases first", + strings.Join(missing, ", ")) + } + return nil +} + +// rawDatabase matches an uncompressed SQLite artifact, including the temporary +// files SQLite leaves mid-build. +var rawDatabase = regexp.MustCompile(`\.db(-journal|-wal|-shm)?$`) + +// checkNoRawDatabases rejects an uncompressed database that reached the output. +// +// The build gzips without keeping the source, so none should exist — but the +// staging directory is copied wholesale, and a leftover from an interrupted run +// would go straight through. A raw database is 100+ MB. +func checkNoRawDatabases(siteDir string) error { + var stray []string + err := filepath.WalkDir(siteDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && rawDatabase.MatchString(d.Name()) { + stray = append(stray, path) + } + return nil + }) + if err != nil { + return err + } + if len(stray) > 0 { + var b strings.Builder + b.WriteString("uncompressed database artefact(s) found in the site output:\n") + for _, f := range stray { + st, _ := os.Stat(f) + fmt.Fprintf(&b, " %s (%.1f MB)\n", f, float64(st.Size())/1048576) + } + b.WriteString("remove them from the staging directory and re-run") + return fmt.Errorf("%s", b.String()) + } + return nil +} + +func copyTree(src, dst string) error { + return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0o755) + } + return copyFile(path, target) + }) +} + +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return err + } + out, err := os.Create(dst) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() +} diff --git a/assembler/internal/site/site_test.go b/assembler/internal/site/site_test.go new file mode 100644 index 0000000..33b0d67 --- /dev/null +++ b/assembler/internal/site/site_test.go @@ -0,0 +1,137 @@ +package site + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tiennm99/thptqg/assembler/internal/registry" +) + +var datasets = []registry.Dataset{{ID: "2016"}, {ID: "2017"}} + +// fakeBuild stands in for a Vite build: an index.html, an asset, and whatever +// databases the caller wants staged. +func fakeBuild(t *testing.T, dbs ...string) Paths { + t.Helper() + root := t.TempDir() + dist := filepath.Join(root, "web", "dist") + write(t, filepath.Join(dist, "index.html"), "app") + write(t, filepath.Join(dist, "assets", "index.js"), "console.log(1)") + for _, name := range dbs { + write(t, filepath.Join(dist, "db", name), "gzipped-bytes") + } + return Paths{Root: root, Web: filepath.Join(root, "web"), Dist: dist, Site: filepath.Join(root, "_site")} +} + +func write(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestAssembleProducesAPageForEveryDataset(t *testing.T) { + p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + if err := Assemble(p, datasets); err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "index.html", + "404.html", + filepath.Join("2016", "index.html"), + filepath.Join("2017", "index.html"), + filepath.Join("assets", "index.js"), + filepath.Join("db", "2016.db.gz"), + } { + if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil { + t.Errorf("missing from the artifact: %s", want) + } + } +} + +// TestMissingDatabaseFailsTheBuild is the guard that closes the widest hole. +// +// Without it the site assembles happily with an empty db/ directory: every page +// renders, every query 404s, and CI stays green. The row-count and size guards +// cannot catch this — they only run when a database was built at all. +func TestMissingDatabaseFailsTheBuild(t *testing.T) { + p := fakeBuild(t, "2016.db.gz") // 2017 never built + err := Assemble(p, datasets) + if err == nil { + t.Fatal("expected an error when a database is missing") + } + if !strings.Contains(err.Error(), "2017.db.gz") { + t.Errorf("the error should name the missing database, got: %v", err) + } +} + +// TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is +// not a database. +func TestEmptyDatabaseFailsTheBuild(t *testing.T) { + p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + write(t, filepath.Join(p.Dist, "db", "2017.db.gz"), "") + if err := Assemble(p, datasets); err == nil { + t.Fatal("expected an error for a zero-byte database") + } +} + +// TestRawDatabaseFailsTheBuild: the compression step deletes its source, so a +// raw .db here means an interrupted run left one behind — and it is 100+ MB. +func TestRawDatabaseFailsTheBuild(t *testing.T) { + for _, name := range []string{"2016.db", "2016.db-journal", "2016.db-wal", "2016.db-shm"} { + t.Run(name, func(t *testing.T) { + p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + write(t, filepath.Join(p.Dist, "db", name), "raw sqlite") + err := Assemble(p, datasets) + if err == nil { + t.Fatalf("expected an error for %s", name) + } + if !strings.Contains(err.Error(), "uncompressed") { + t.Errorf("unexpected error: %v", err) + } + }) + } +} + +// TestGzipIsNotMistakenForRaw: the reject pattern is anchored, so a .db.gz must +// pass. Getting this wrong would fail every build. +func TestGzipIsNotMistakenForRaw(t *testing.T) { + if rawDatabase.MatchString("2016.db.gz") { + t.Error("a .db.gz must not be treated as an uncompressed database") + } + for _, name := range []string{"2016.db", "x.db-journal", "x.db-wal", "x.db-shm"} { + if !rawDatabase.MatchString(name) { + t.Errorf("%s should be treated as an uncompressed artifact", name) + } + } +} + +func TestAssembleRejectsAMissingBuild(t *testing.T) { + root := t.TempDir() + p := Paths{Root: root, Web: root, Dist: filepath.Join(root, "dist"), Site: filepath.Join(root, "_site")} + if err := Assemble(p, datasets); err == nil { + t.Fatal("expected an error when there is no Vite build") + } +} + +// TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a +// previous run's leftovers cannot survive into the artifact. +func TestAssembleIsIdempotent(t *testing.T) { + p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + if err := Assemble(p, datasets); err != nil { + t.Fatal(err) + } + stale := filepath.Join(p.Site, "2015", "index.html") + write(t, stale, "old dataset") + if err := Assemble(p, datasets); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Error("a directory from a previous run survived into the new artifact") + } +} diff --git a/crawler/cmd/crawl/main.go b/crawler/cmd/crawl/main.go new file mode 100644 index 0000000..c6c394a --- /dev/null +++ b/crawler/cmd/crawl/main.go @@ -0,0 +1,153 @@ +// Command crawl downloads a dataset's source spreadsheets into data//. +// +// The argument is the dataset id, the same one used by the parser's configs and +// the published site paths. +// +// crawl 2016 # 119 exam-cluster files +// crawl 2017 # 63 province files +// crawl 2017 --list # show what would be downloaded +// +// Each run reads the download links out of the article that published them, so +// --list needs network access too. Runs are idempotent: a file already present +// and non-empty is skipped, so an interrupted crawl can simply be re-run. +package main + +import ( + "context" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/tiennm99/thptqg/crawler/internal/article" + "github.com/tiennm99/thptqg/crawler/internal/fetch" + "github.com/tiennm99/thptqg/crawler/internal/sources" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "crawl: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, "usage: crawl [flags]\n\nDatasets:\n") + for _, s := range sources.All() { + fmt.Fprintf(os.Stderr, " %-6s %s\n", s.ID, s.Summary) + } + fmt.Fprint(os.Stderr, "\nFlags:\n") + fmt.Fprint(os.Stderr, " --out string output directory (default ../data/)\n") + fmt.Fprint(os.Stderr, " --concurrency int parallel downloads (default 6)\n") + fmt.Fprint(os.Stderr, " --timeout duration per-file timeout (default 2m)\n") + fmt.Fprint(os.Stderr, " --list print the file list and exit\n") +} + +func run(args []string) error { + if len(args) == 0 || args[0] == "-h" || args[0] == "--help" { + usage() + if len(args) == 0 { + return fmt.Errorf("no dataset given") + } + return nil + } + + src, err := sources.Lookup(args[0]) + if err != nil { + usage() + return err + } + + fs := flag.NewFlagSet(src.ID, flag.ContinueOnError) + // The default is relative to the crawler module directory, which is where + // both `go -C crawler run ./cmd/crawl` and a manual `cd crawler` land. + out := fs.String("out", filepath.Join("..", "data", src.ID), "output directory") + concurrency := fs.Int("concurrency", 6, "parallel downloads") + timeout := fs.Duration("timeout", 2*time.Minute, "per-file timeout") + list := fs.Bool("list", false, "print the file list and exit") + if err := fs.Parse(args[1:]); err != nil { + return err + } + + // Ctrl-C cancels the article fetch and any in-flight download; each worker + // deletes its partial file on the way out, so an interrupted run leaves + // nothing half-written. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + client := &http.Client{Timeout: *timeout} + + fmt.Printf("Reading %s\n", src.Article) + links, err := article.Fetch(ctx, client, src.Article, src.Headers, src.Exts...) + if err != nil { + return err + } + files, err := src.Resolve(links) + if err != nil { + return err + } + + outDir, err := filepath.Abs(*out) + if err != nil { + return err + } + + items := make([]fetch.Item, 0, len(files)) + for _, f := range files { + items = append(items, fetch.Item{ + Name: f.Name, + URL: f.URL, + Path: filepath.Join(outDir, f.Dest), + }) + } + + if *list { + for _, it := range items { + fmt.Printf("%s\t%s\n", filepath.Base(it.Path), it.URL) + } + return nil + } + + fmt.Printf("Downloading %d files to %s...\n", len(items), outDir) + + results, runErr := fetch.Run(ctx, items, fetch.Options{ + Concurrency: *concurrency, + Timeout: *timeout, + Headers: src.Headers, + OnResult: printResult, + }) + if runErr != nil { + return runErr + } + + ok, skip, failed := fetch.Tally(results) + fmt.Printf("\nDone. ok=%d skip=%d fail=%d\n", ok, skip, len(failed)) + if len(failed) > 0 { + for _, r := range failed { + fmt.Fprintf(os.Stderr, " %s: %v\n", r.Item.Name, r.Err) + } + return fmt.Errorf("%d file(s) failed", len(failed)) + } + return nil +} + +func printResult(done, total int, r fetch.Result) { + tag := map[fetch.Status]string{ + fetch.StatusOK: "✓", + fetch.StatusSkip: "·", + fetch.StatusFail: "✗", + }[r.Status] + + detail := "" + switch r.Status { + case fetch.StatusFail: + detail = r.Err.Error() + default: + detail = fmt.Sprintf("%.0f KB", float64(r.Size)/1024) + } + fmt.Printf(" %s [%d/%d] %-20s %s\n", tag, done, total, r.Item.Name, detail) +} diff --git a/crawler/go.mod b/crawler/go.mod new file mode 100644 index 0000000..218e045 --- /dev/null +++ b/crawler/go.mod @@ -0,0 +1,8 @@ +module github.com/tiennm99/thptqg/crawler + +go 1.26.5 + +require ( + golang.org/x/net v0.58.0 + golang.org/x/text v0.41.0 +) diff --git a/crawler/go.sum b/crawler/go.sum new file mode 100644 index 0000000..cc9727d --- /dev/null +++ b/crawler/go.sum @@ -0,0 +1,4 @@ +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= diff --git a/crawler/internal/article/article.go b/crawler/internal/article/article.go new file mode 100644 index 0000000..c808d2b --- /dev/null +++ b/crawler/internal/article/article.go @@ -0,0 +1,144 @@ +// Package article reads download links out of a published web page. +// +// This is what makes the crawler a crawler: the file lists are not carried in +// the repository, they are read from the articles that published them, at run +// time. A source declares which page to read and how to name what it finds. +package article + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "golang.org/x/net/html" +) + +// Link is one downloadable file found on a page. +type Link struct { + // URL is absolute, resolved against the page it was found on. + URL string + // Text is the anchor's visible text, whitespace-collapsed. Some pages use + // it for the thing being linked ("Bạc Liêu"), others for boilerplate + // ("xem TẠI ĐÂY"), so a source decides whether it is worth anything. + Text string + // File is the last segment of the URL path, query string excluded. + File string +} + +// Extract returns every anchor on the page whose target ends in one of exts. +// +// Pure: no network, so the parsing rules can be tested against a saved copy of +// a real page. Order is document order, and duplicates are kept — deciding +// whether two links to the same file is a problem belongs to the caller, which +// knows what it would name them. +func Extract(pageURL string, body io.Reader, exts ...string) ([]Link, error) { + base, err := url.Parse(pageURL) + if err != nil { + return nil, fmt.Errorf("bad page URL %q: %w", pageURL, err) + } + + doc, err := html.Parse(body) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", pageURL, err) + } + + var out []Link + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + if link, ok := linkOf(base, n, exts); ok { + out = append(out, link) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + + return out, nil +} + +// linkOf turns one element into a Link, or reports that it is not one of +// the files being looked for. +func linkOf(base *url.URL, n *html.Node, exts []string) (Link, bool) { + var href string + for _, a := range n.Attr { + if strings.EqualFold(a.Key, "href") { + href = strings.TrimSpace(a.Val) + break + } + } + if href == "" { + return Link{}, false + } + + // Relative hrefs are the norm on the pages this reads; resolving against + // the page URL is what lets a source name a mirror and get its files. + ref, err := url.Parse(href) + if err != nil { + return Link{}, false + } + abs := base.ResolveReference(ref) + + file := path.Base(abs.Path) + if !hasExt(file, exts) { + return Link{}, false + } + + return Link{URL: abs.String(), Text: textOf(n), File: file}, true +} + +func hasExt(name string, exts []string) bool { + lower := strings.ToLower(name) + for _, e := range exts { + if strings.HasSuffix(lower, strings.ToLower(e)) { + return true + } + } + return false +} + +// textOf collects an element's visible text, collapsing whitespace. Anchors on +// these pages wrap the label in styling tags, so the text is rarely a single +// child node. +func textOf(n *html.Node) string { + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.Join(strings.Fields(b.String()), " ") +} + +// Fetch downloads a page and runs Extract on it. +func Fetch(ctx context.Context, client *http.Client, pageURL string, headers map[string]string, exts ...string) ([]Link, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return nil, err + } + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", pageURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch %s: HTTP %d", pageURL, resp.StatusCode) + } + + return Extract(pageURL, resp.Body, exts...) +} diff --git a/crawler/internal/article/article_test.go b/crawler/internal/article/article_test.go new file mode 100644 index 0000000..6bcb0f5 --- /dev/null +++ b/crawler/internal/article/article_test.go @@ -0,0 +1,152 @@ +package article + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const page = "https://example.test/news/scores.html" + +func extract(t *testing.T, body string, exts ...string) []Link { + t.Helper() + links, err := Extract(page, strings.NewReader(body), exts...) + if err != nil { + t.Fatal(err) + } + return links +} + +// TestResolvesRelativeHrefs is the behaviour the 2016 source depends on: its +// article links files as /upload/..., so they only become fetchable once +// resolved against the page they were found on. +func TestResolvesRelativeHrefs(t *testing.T) { + got := extract(t, ` + root-relative + page-relative + already absolute + protocol-relative + `, ".xlsx") + + want := []string{ + "https://example.test/upload/s/a.xlsx", + "https://example.test/news/sub/b.xlsx", + "https://cdn.other/c.xlsx", + "https://cdn.other/d.xlsx", + } + if len(got) != len(want) { + t.Fatalf("got %d links, want %d", len(got), len(want)) + } + for i, w := range want { + if got[i].URL != w { + t.Errorf("link %d = %q, want %q", i, got[i].URL, w) + } + } +} + +func TestFiltersByExtension(t *testing.T) { + got := extract(t, ` + keep + keep + drop + drop + keep, case-insensitive + no href at all + `, ".xls", ".xlsx") + + if len(got) != 3 { + t.Fatalf("got %d links, want 3: %+v", len(got), got) + } + // ".xls" must not swallow ".xlsx" or vice versa when only one is asked for. + only := extract(t, `xy`, ".xlsx") + if len(only) != 1 || only[0].File != "b.xlsx" { + t.Errorf("extension filter is too loose: %+v", only) + } +} + +// TestFileIgnoresQueryString: a query string is not part of the filename, and +// writing one to disk would produce a name the parser never sees. +func TestFileIgnoresQueryString(t *testing.T) { + got := extract(t, `x`, ".xlsx") + if len(got) != 1 { + t.Fatalf("got %d links", len(got)) + } + if got[0].File != "report.xlsx" { + t.Errorf("File = %q, want report.xlsx", got[0].File) + } + if !strings.Contains(got[0].URL, "v=2") { + t.Errorf("the query string must survive in the URL: %q", got[0].URL) + } +} + +// TestTextCollectsNestedMarkup: both real articles wrap the label in styling +// tags, so the anchor text is never a single child node. 2017 names its files +// from this text. +func TestTextCollectsNestedMarkup(t *testing.T) { + got := extract(t, `Bà Rịa -Vũng Tàu`, ".xls") + if len(got) != 1 { + t.Fatalf("got %d links", len(got)) + } + if got[0].Text != "Bà Rịa -Vũng Tàu" { + t.Errorf("Text = %q, want %q", got[0].Text, "Bà Rịa -Vũng Tàu") + } +} + +func TestEntitiesInHrefAreDecoded(t *testing.T) { + got := extract(t, `x`, ".xlsx") + if len(got) != 1 || !strings.Contains(got[0].URL, "x=1&y=2") { + t.Errorf("href entity not decoded: %+v", got) + } +} + +// TestMalformedHTMLStillParses: these are hand-edited CMS pages, and the parser +// must not give up on unclosed tags. +func TestMalformedHTMLStillParses(t *testing.T) { + got := extract(t, `

listonetwo`, ".xls") + if len(got) != 2 { + t.Errorf("got %d links, want 2: %+v", len(got), got) + } +} + +func TestExtractRejectsBadPageURL(t *testing.T) { + if _, err := Extract("://nonsense", strings.NewReader("x"), ".xls"); err == nil { + t.Error("expected an error for an unparseable page URL") + } +} + +func TestFetchSendsHeadersAndParses(t *testing.T) { + var ua, ref string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua, ref = r.Header.Get("User-Agent"), r.Header.Get("Referer") + w.Write([]byte(`An Giang`)) + })) + defer srv.Close() + + links, err := Fetch(t.Context(), srv.Client(), srv.URL, + map[string]string{"User-Agent": "test-agent", "Referer": "https://ref.test/"}, ".xls") + if err != nil { + t.Fatal(err) + } + if ua != "test-agent" || ref != "https://ref.test/" { + t.Errorf("headers not sent: ua=%q referer=%q", ua, ref) + } + if len(links) != 1 || links[0].Text != "An Giang" || links[0].File != "a.xls" { + t.Errorf("unexpected links: %+v", links) + } +} + +// TestFetchFailsLoudly: a CMS that answers 404 or 403 with an HTML error page +// would otherwise yield zero links and read as "nothing to download". +func TestFetchFailsLoudly(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "not found", http.StatusNotFound) + })) + defer srv.Close() + + if _, err := Fetch(t.Context(), srv.Client(), srv.URL, nil, ".xls"); err == nil { + t.Fatal("expected an error for a non-200 response") + } else if !strings.Contains(err.Error(), "404") { + t.Errorf("error should name the status, got: %v", err) + } +} diff --git a/crawler/internal/fetch/fetch.go b/crawler/internal/fetch/fetch.go new file mode 100644 index 0000000..292f72e --- /dev/null +++ b/crawler/internal/fetch/fetch.go @@ -0,0 +1,209 @@ +// Package fetch downloads a list of files concurrently, skipping any that are +// already on disk. +// +// It is deliberately source-agnostic: it knows nothing about provinces, exam +// years or spreadsheet formats. Each source in internal/sources produces a +// []Item and this package moves the bytes. +package fetch + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +// Item is one file to download. +type Item struct { + // Name is the human label used in progress output, e.g. "An Giang". + Name string + URL string + // Path is the absolute destination, including filename. + Path string +} + +// Status is the outcome of one item. +type Status int + +const ( + // StatusOK means the file was downloaded during this run. + StatusOK Status = iota + // StatusSkip means a non-empty file was already present. + StatusSkip + // StatusFail means the download did not complete; Result.Err says why. + StatusFail +) + +// Result records what happened to one item. +type Result struct { + Item Item + Status Status + Size int64 + Err error +} + +// Options configures a run. The zero value is usable: Concurrency and Timeout +// fall back to defaults. +type Options struct { + // Concurrency is the number of files downloaded at once. + Concurrency int + // Timeout bounds each individual request, headers and body together. + // The original JS crawler had no timeout, so one stalled connection could + // hang the whole run indefinitely. + Timeout time.Duration + // Headers are sent with every request. The CDN this was written against + // rejects requests without a browser User-Agent and a matching Referer. + Headers map[string]string + // OnResult, if set, is called once per completed item. Calls are + // serialised, so it does not need its own locking, but they arrive in + // completion order rather than list order. + OnResult func(done, total int, r Result) +} + +const ( + defaultConcurrency = 6 + defaultTimeout = 2 * time.Minute +) + +// Run downloads every item, returning one Result each. It does not return an +// error for a failed download — that is reported per item — only for a problem +// that stops the run as a whole, such as an unusable output directory. +// +// Results come back in completion order, not input order. +func Run(ctx context.Context, items []Item, opts Options) ([]Result, error) { + if opts.Concurrency <= 0 { + opts.Concurrency = defaultConcurrency + } + if opts.Timeout <= 0 { + opts.Timeout = defaultTimeout + } + + // Every item's parent directory must exist before any worker starts, so a + // missing output directory fails once here rather than N times in parallel. + for _, it := range items { + if err := os.MkdirAll(filepath.Dir(it.Path), 0o755); err != nil { + return nil, fmt.Errorf("cannot create output directory: %w", err) + } + } + + client := &http.Client{Timeout: opts.Timeout} + + var ( + mu sync.Mutex + results = make([]Result, 0, len(items)) + next int + ) + + work := make(chan Item) + var wg sync.WaitGroup + for range opts.Concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for it := range work { + r := download(ctx, client, it, opts.Headers) + mu.Lock() + results = append(results, r) + next++ + if opts.OnResult != nil { + opts.OnResult(next, len(items), r) + } + mu.Unlock() + } + }() + } + + for _, it := range items { + select { + case work <- it: + case <-ctx.Done(): + close(work) + wg.Wait() + return results, ctx.Err() + } + } + close(work) + wg.Wait() + + return results, nil +} + +// download fetches one item, or reports that it was already present. +// +// The body is streamed to a .part file and renamed into place only once it is +// complete. Without that, an interrupted run leaves a truncated file at the +// final path — and because the skip check only tests for a non-empty file, +// every later run would skip it and the corruption would persist silently. +// +// A .part left behind by an abrupt kill needs no cleanup: it does not satisfy +// the skip check, os.Create truncates it, and the next run re-fetches the file. +// The parser ignores it in the meantime, since it reads only .xls and .xlsx. +func download(ctx context.Context, client *http.Client, it Item, headers map[string]string) Result { + if st, err := os.Stat(it.Path); err == nil && st.Size() > 0 { + return Result{Item: it, Status: StatusSkip, Size: st.Size()} + } + + fail := func(err error) Result { + return Result{Item: it, Status: StatusFail, Err: err} + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, it.URL, nil) + if err != nil { + return fail(err) + } + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := client.Do(req) + if err != nil { + return fail(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fail(fmt.Errorf("HTTP %d", resp.StatusCode)) + } + + part := it.Path + ".part" + f, err := os.Create(part) + if err != nil { + return fail(err) + } + n, err := io.Copy(f, resp.Body) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(part) + return fail(err) + } + if n == 0 { + os.Remove(part) + return fail(fmt.Errorf("empty response body")) + } + if err := os.Rename(part, it.Path); err != nil { + os.Remove(part) + return fail(err) + } + + return Result{Item: it, Status: StatusOK, Size: n} +} + +// Tally counts results by status. +func Tally(results []Result) (ok, skip int, failed []Result) { + for _, r := range results { + switch r.Status { + case StatusOK: + ok++ + case StatusSkip: + skip++ + case StatusFail: + failed = append(failed, r) + } + } + return ok, skip, failed +} diff --git a/crawler/internal/fetch/fetch_test.go b/crawler/internal/fetch/fetch_test.go new file mode 100644 index 0000000..f4214a1 --- /dev/null +++ b/crawler/internal/fetch/fetch_test.go @@ -0,0 +1,184 @@ +package fetch + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func serve(t *testing.T, h http.HandlerFunc) string { + t.Helper() + s := httptest.NewServer(h) + t.Cleanup(s.Close) + return s.URL +} + +func TestDownloadsAndSkips(t *testing.T) { + var hits int + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + hits++ + w.Write([]byte("payload")) + }) + + dir := t.TempDir() + items := []Item{{Name: "one", URL: url, Path: filepath.Join(dir, "one.xls")}} + + results, err := Run(context.Background(), items, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusOK { + t.Fatalf("status = %v, err = %v", results[0].Status, results[0].Err) + } + if got, _ := os.ReadFile(items[0].Path); string(got) != "payload" { + t.Errorf("content = %q", got) + } + + // Second run must not re-fetch. + results, err = Run(context.Background(), items, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusSkip { + t.Errorf("status = %v, want StatusSkip", results[0].Status) + } + if hits != 1 { + t.Errorf("server hit %d times, want 1", hits) + } +} + +func TestNon200Fails(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "gone", http.StatusNotFound) + }) + + dir := t.TempDir() + path := filepath.Join(dir, "missing.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Fatalf("status = %v, want StatusFail", results[0].Status) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Error("a failed download must leave no file at the destination") + } + assertNoPartFiles(t, dir) +} + +// TestFailureLeavesNoPartial is the reason downloads land on a .part file +// first. Writing straight to the destination would leave a truncated file +// there, and the skip check — which only tests for a non-empty file — would +// skip it on every later run, so the corruption would never be re-fetched. +func TestFailureLeavesNoPartial(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1000") + w.Write([]byte("short")) + // Closing early makes the body read fail mid-copy. + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) + }) + + dir := t.TempDir() + path := filepath.Join(dir, "truncated.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Fatalf("status = %v, want StatusFail", results[0].Status) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Error("a truncated download must not be left at the destination") + } + assertNoPartFiles(t, dir) +} + +func TestEmptyBodyFails(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) {}) + + dir := t.TempDir() + path := filepath.Join(dir, "empty.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Errorf("an empty body must fail, got %v", results[0].Status) + } + assertNoPartFiles(t, dir) +} + +func TestHeadersAreSent(t *testing.T) { + var gotUA, gotRef string + url := serve(t, func(w http.ResponseWriter, r *http.Request) { + gotUA, gotRef = r.Header.Get("User-Agent"), r.Header.Get("Referer") + w.Write([]byte("ok")) + }) + + _, err := Run(context.Background(), + []Item{{Name: "x", URL: url, Path: filepath.Join(t.TempDir(), "x.xls")}}, + Options{Headers: map[string]string{"User-Agent": "test-agent", "Referer": "https://example.test/"}}) + if err != nil { + t.Fatal(err) + } + if gotUA != "test-agent" || gotRef != "https://example.test/" { + t.Errorf("headers not sent: ua=%q referer=%q", gotUA, gotRef) + } +} + +func TestAllItemsRun(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("x")) + }) + + dir := t.TempDir() + var items []Item + for i := range 20 { + items = append(items, Item{ + Name: "f", + URL: url, + Path: filepath.Join(dir, "sub", string(rune('a'+i))+".xls"), + }) + } + + results, err := Run(context.Background(), items, Options{Concurrency: 4}) + if err != nil { + t.Fatal(err) + } + if len(results) != len(items) { + t.Fatalf("got %d results, want %d", len(results), len(items)) + } + ok, _, failed := Tally(results) + if ok != len(items) { + t.Errorf("ok = %d, want %d (failures: %v)", ok, len(items), failed) + } +} + +func TestTally(t *testing.T) { + ok, skip, failed := Tally([]Result{ + {Status: StatusOK}, {Status: StatusOK}, {Status: StatusSkip}, {Status: StatusFail}, + }) + if ok != 2 || skip != 1 || len(failed) != 1 { + t.Errorf("ok=%d skip=%d fail=%d", ok, skip, len(failed)) + } +} + +func assertNoPartFiles(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if filepath.Ext(e.Name()) == ".part" { + t.Errorf("leftover partial file: %s", e.Name()) + } + } +} diff --git a/crawler/internal/sources/source_2016.go b/crawler/internal/sources/source_2016.go new file mode 100644 index 0000000..3e2f1a8 --- /dev/null +++ b/crawler/internal/sources/source_2016.go @@ -0,0 +1,40 @@ +package sources + +import ( + "fmt" + + "github.com/tiennm99/thptqg/crawler/internal/article" +) + +// source2016 fetches the 2016 dataset: one spreadsheet per exam cluster +// (cụm thi), 4 .xls and 115 .xlsx. +// +// The article is a mirror. The site that first published this list +// (dtntbacgiang.edu.vn) no longer resolves; this copy of the same article is +// still online, and its links are read at crawl time like any other source. +// +// Dest keeps the server's own filename verbatim — a 32-hex content hash, the +// cluster slug, then a millisecond timestamp. Two reasons not to prettify it: +// +// - parser sorts inputs bytewise and inserts last-wins, so filenames decide +// which row survives a duplicate exam number. That is live here, not +// hypothetical: 877,464 source rows collapse to 877,461, so three rows' +// contents depend on this ordering. +// - parser/testdata/reader-fidelity-hashes.tsv is keyed by full path, and +// it is frozen — it was produced by the Rust reader, which no longer exists. +// +// Unlike 2017 this needs no transliteration: the name comes from the URL, so it +// is exact by construction rather than derived from a label. +var source2016 = Source{ + ID: "2016", + Summary: "119 exam-cluster files (4 .xls + 115 .xlsx)", + Article: "https://dtnt.bacninh.edu.vn/tin-tuc/tin-tuc-su-kien/cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html", + Exts: []string{".xls", ".xlsx"}, + WantFiles: 119, + Dest: func(l article.Link) (string, error) { + if l.File == "" { + return "", fmt.Errorf("link %s has no filename", l.URL) + } + return l.File, nil + }, +} diff --git a/crawler/internal/sources/source_2017.go b/crawler/internal/sources/source_2017.go new file mode 100644 index 0000000..d6abb9e --- /dev/null +++ b/crawler/internal/sources/source_2017.go @@ -0,0 +1,70 @@ +package sources + +import ( + "fmt" + "regexp" + "strings" + + "golang.org/x/text/unicode/norm" + + "github.com/tiennm99/thptqg/crawler/internal/article" +) + +// browserUA and the Referer are both required: the CDN 403s an unrecognised +// User-Agent, and rejects requests that do not carry the article as Referer. +const browserUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" + +const article2017 = "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm" + +// source2017 fetches the 2017 dataset: one .xls per province, still served from +// the CDN that originally published it. +// +// Dest cannot use the CDN's filenames the way 2016 does, because they are +// inconsistent — Angiang.xls, 1BaRiaVungTau.xls, 23HaiPhong.xls, Gia-Lai.xls — +// carrying upload-order prefixes and arbitrary capitalisation. The province name +// in the link text is the stable identifier, so the local name is derived from +// that instead, which is what produced the files in data/2017/. +var source2017 = Source{ + ID: "2017", + Summary: "63 province .xls files from the baotintuc.vn CDN", + Article: article2017, + Exts: []string{".xls"}, + WantFiles: 63, + Headers: map[string]string{ + "User-Agent": browserUA, + "Referer": article2017, + }, + Dest: func(l article.Link) (string, error) { + s := slug(l.Text) + if s == "" { + return "", fmt.Errorf("link %s has no usable label to name it by", l.URL) + } + return s + ".xls", nil + }, +} + +var nonAlphanumeric = regexp.MustCompile(`[^a-z0-9]+`) + +// slug turns a province name into a filename stem: transliterated to ASCII, +// lowercased, with every run of other characters collapsed to one hyphen. +// +// "Bà Rịa -Vũng Tàu" -> "ba-ria-vung-tau" +// "Bắc Kạn" -> "bac-kan" +// +// The article writes the names with diacritics, so this has to strip them. +// Combining marks are removed by the literal range U+0300–U+036F rather than by +// the unicode.Mn category, matching what parser does to build ho_ten_ascii. +func slug(name string) string { + var b strings.Builder + for _, r := range norm.NFD.String(name) { + switch { + case r >= 0x0300 && r <= 0x036F: + // combining mark: drop + case r == 'đ' || r == 'Đ': + b.WriteByte('d') + default: + b.WriteRune(r) + } + } + return strings.Trim(nonAlphanumeric.ReplaceAllString(strings.ToLower(b.String()), "-"), "-") +} diff --git a/crawler/internal/sources/sources.go b/crawler/internal/sources/sources.go new file mode 100644 index 0000000..5ddaf5d --- /dev/null +++ b/crawler/internal/sources/sources.go @@ -0,0 +1,105 @@ +// Package sources says where each dataset's spreadsheets are published and +// what to call them locally. +// +// It carries no link lists. Each source names the article that published the +// files; internal/article reads the links out of that page at run time, and +// internal/fetch moves the bytes. +package sources + +import ( + "fmt" + + "github.com/tiennm99/thptqg/crawler/internal/article" +) + +// Source is one crawlable dataset. +type Source struct { + // ID is the dataset id: the subcommand, the directory under data/, and the + // parser config name, all at once. Keeping it single means a source + // cannot be pointed at the wrong dataset's directory. + ID string + Summary string + + // Article is the published page listing this dataset's files. + Article string + + // Exts are the file extensions to pick out of that page. + Exts []string + + // Headers are sent with every request for this source, for the article and + // the files alike. + Headers map[string]string + + // WantFiles is how many links the article is expected to yield. A page that + // suddenly yields fewer has changed shape, and silently crawling a partial + // dataset is the failure this exists to prevent — parser would happily + // build a short database and only the row-count guard would catch it, after + // the fact. + WantFiles int + + // Dest names the local file for one discovered link. + // + // This is the load-bearing part. parser sorts input files bytewise and + // inserts last-wins, so the names chosen here decide which row survives a + // duplicate exam number. Two sources answer it differently and both have a + // reason: see source_2016.go and source_2017.go. + Dest func(article.Link) (string, error) +} + +// File is one spreadsheet to download. +type File struct { + // Name is the human label used in progress output. + Name string + URL string + // Dest is the filename within the dataset directory. + Dest string +} + +// Resolve turns the links found on a source's article into the files to fetch, +// enforcing the count and rejecting any two links that would write to the same +// name — which would silently cost the dataset a file. +func (s Source) Resolve(links []article.Link) ([]File, error) { + if len(links) != s.WantFiles { + return nil, fmt.Errorf( + "%s: found %d links on %s, expected %d — the page has changed shape; "+ + "check it before crawling, a partial dataset builds without complaint", + s.ID, len(links), s.Article, s.WantFiles) + } + + out := make([]File, 0, len(links)) + seen := make(map[string]string, len(links)) + for _, l := range links { + dest, err := s.Dest(l) + if err != nil { + return nil, fmt.Errorf("%s: %w", s.ID, err) + } + if prev, dup := seen[dest]; dup { + return nil, fmt.Errorf("%s: %q and %q both name the local file %q", + s.ID, prev, l.URL, dest) + } + seen[dest] = l.URL + + name := l.Text + if name == "" { + name = dest + } + out = append(out, File{Name: name, URL: l.URL, Dest: dest}) + } + return out, nil +} + +// registry is ordered: it drives the help output. +var registry = []Source{source2016, source2017} + +// All returns every known source, in help order. +func All() []Source { return registry } + +// Lookup finds a source by ID. +func Lookup(id string) (Source, error) { + for _, s := range registry { + if s.ID == id { + return s, nil + } + } + return Source{}, fmt.Errorf("unknown dataset %q", id) +} diff --git a/crawler/internal/sources/sources_test.go b/crawler/internal/sources/sources_test.go new file mode 100644 index 0000000..593ea18 --- /dev/null +++ b/crawler/internal/sources/sources_test.go @@ -0,0 +1,192 @@ +package sources + +import ( + "compress/gzip" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tiennm99/thptqg/crawler/internal/article" +) + +// The fixtures are saved copies of the two source articles, kept so the link +// extraction and the naming rules can be exercised without the network. +// +// article-2017 is the live page. article-2016 came from the Internet Archive's +// copy of the site that first published that list (dtntbacgiang.edu.vn, which +// no longer resolves); the mirror the crawler actually reads carries the same +// article. Its hrefs are relative, so they resolve against whichever host the +// source names, and the filenames are identical either way. +func fixture(t *testing.T, id string) *gzip.Reader { + t.Helper() + f, err := os.Open(filepath.Join("testdata", "article-"+id+".html.gz")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { gz.Close() }) + return gz +} + +// resolveFixture runs a source's full pipeline over its saved article. +func resolveFixture(t *testing.T, src Source) []File { + t.Helper() + links, err := article.Extract(src.Article, fixture(t, src.ID), src.Exts...) + if err != nil { + t.Fatal(err) + } + files, err := src.Resolve(links) + if err != nil { + t.Fatal(err) + } + return files +} + +// TestReproducesFilesOnDisk is the guard that matters. +// +// parser sorts its inputs and inserts last-wins, so filenames decide which +// row survives a duplicate exam number. If extraction or naming drifted, a +// re-crawl could rebuild a database with the same row count and different +// content, which the row-count guard in build-db.js would not catch. +// +// The committed data// is the oracle: reading the source article and +// applying the source's naming rule must reproduce it exactly, both directions. +func TestReproducesFilesOnDisk(t *testing.T) { + for _, src := range All() { + t.Run(src.ID, func(t *testing.T) { + dir := filepath.Join("..", "..", "..", "data", src.ID) + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + t.Skipf("%s not present", dir) + } + if err != nil { + t.Fatal(err) + } + onDisk := map[string]bool{} + for _, e := range entries { + if !e.IsDir() { + onDisk[e.Name()] = true + } + } + + for _, f := range resolveFixture(t, src) { + if !onDisk[f.Dest] { + t.Errorf("crawler would write %q, which is not in %s — "+ + "a renamed input can change which duplicate row survives", f.Dest, dir) + } + delete(onDisk, f.Dest) + } + for name := range onDisk { + t.Errorf("%s/%s exists but the article produces no link for it", dir, name) + } + }) + } +} + +// TestResolvedURLsAreAbsolute: 2016's article uses relative hrefs, so a +// mis-resolved base would produce URLs that cannot be fetched at all. +func TestResolvedURLsAreAbsolute(t *testing.T) { + for _, src := range All() { + t.Run(src.ID, func(t *testing.T) { + for _, f := range resolveFixture(t, src) { + if !strings.HasPrefix(f.URL, "https://") { + t.Errorf("%s: %q is not an absolute https URL", f.Dest, f.URL) + } + } + }) + } +} + +func TestExtensionsMatchWhatIsExpected(t *testing.T) { + counts := map[string]map[string]int{} + for _, src := range All() { + c := map[string]int{} + for _, f := range resolveFixture(t, src) { + c[filepath.Ext(f.Dest)]++ + } + counts[src.ID] = c + } + if got := counts["2016"]; got[".xls"] != 4 || got[".xlsx"] != 115 { + t.Errorf("2016: %d .xls and %d .xlsx, want 4 and 115", got[".xls"], got[".xlsx"]) + } + if got := counts["2017"]; got[".xls"] != 63 { + t.Errorf("2017: %d .xls, want 63", got[".xls"]) + } +} + +// TestResolveRejectsShortPage: a page that has changed shape must stop the +// crawl, not quietly produce a partial dataset. +func TestResolveRejectsShortPage(t *testing.T) { + src := source2017 + _, err := src.Resolve([]article.Link{{URL: "https://x/a.xls", Text: "An Giang", File: "a.xls"}}) + if err == nil { + t.Fatal("expected an error for a short link list") + } + if !strings.Contains(err.Error(), "expected 63") { + t.Errorf("error should name the expected count, got: %v", err) + } +} + +// TestResolveRejectsCollidingNames: two links naming the same local file would +// cost the dataset a file with no error anywhere downstream. +func TestResolveRejectsCollidingNames(t *testing.T) { + src := Source{ + ID: "t", WantFiles: 2, + Dest: func(article.Link) (string, error) { return "same.xls", nil }, + } + _, err := src.Resolve([]article.Link{ + {URL: "https://x/a.xls", File: "a.xls"}, + {URL: "https://x/b.xls", File: "b.xls"}, + }) + if err == nil || !strings.Contains(err.Error(), "same.xls") { + t.Errorf("expected a collision error naming the file, got: %v", err) + } +} + +func TestSlug(t *testing.T) { + // The article writes province names with diacritics; the committed + // filenames are ASCII. These are the awkward ones. + for in, want := range map[string]string{ + "An Giang": "an-giang", + "Bạc Liêu": "bac-lieu", + "Bắc Kạn": "bac-kan", + "Bà Rịa -Vũng Tàu": "ba-ria-vung-tau", + "Thừa Thiên - Huế": "thua-thien-hue", + "Đắk Lắk": "dak-lak", + } { + if got := slug(in); got != want { + t.Errorf("slug(%q) = %q, want %q", in, got, want) + } + } +} + +func TestLookup(t *testing.T) { + for _, s := range All() { + got, err := Lookup(s.ID) + if err != nil { + t.Errorf("Lookup(%q): %v", s.ID, err) + } + if got.ID != s.ID || got.Article == "" || got.WantFiles == 0 { + t.Errorf("Lookup(%q) returned an incomplete source: %+v", s.ID, got) + } + } + if _, err := Lookup("nope"); err == nil { + t.Error("Lookup of an unknown dataset must fail") + } +} + +// TestIDsAreDatasetIDs: the ID doubles as the directory under data/, so a typo +// would send a crawl into a directory the parser never reads. +func TestIDsAreDatasetIDs(t *testing.T) { + for _, s := range All() { + dir := filepath.Join("..", "..", "..", "data", s.ID) + if _, err := os.Stat(dir); os.IsNotExist(err) { + t.Errorf("source %q has no dataset directory at %s", s.ID, dir) + } + } +} diff --git a/crawler/internal/sources/testdata/article-2016.html.gz b/crawler/internal/sources/testdata/article-2016.html.gz new file mode 100644 index 0000000..3c60dc0 Binary files /dev/null and b/crawler/internal/sources/testdata/article-2016.html.gz differ diff --git a/crawler/internal/sources/testdata/article-2017.html.gz b/crawler/internal/sources/testdata/article-2017.html.gz new file mode 100644 index 0000000..b8b87d4 Binary files /dev/null and b/crawler/internal/sources/testdata/article-2017.html.gz differ diff --git a/data/2017-old/10_BinhThuan_RIIW.xls.xlsx b/data/2017-old/10_BinhThuan_RIIW.xls.xlsx deleted file mode 100644 index 8ecaccd..0000000 Binary files a/data/2017-old/10_BinhThuan_RIIW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx b/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx deleted file mode 100644 index 8fe2b4a..0000000 Binary files a/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_LamDong_GNFT.xls.xlsx b/data/2017-old/10_LamDong_GNFT.xls.xlsx deleted file mode 100644 index 54b89a4..0000000 Binary files a/data/2017-old/10_LamDong_GNFT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx b/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx deleted file mode 100644 index 70a0ff8..0000000 Binary files a/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/11_BinhDuong_RYQL.xls.xlsx b/data/2017-old/11_BinhDuong_RYQL.xls.xlsx deleted file mode 100644 index ccb2951..0000000 Binary files a/data/2017-old/11_BinhDuong_RYQL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/11_LaoCai_GMSU.xls.xlsx b/data/2017-old/11_LaoCai_GMSU.xls.xlsx deleted file mode 100644 index 33ed6f0..0000000 Binary files a/data/2017-old/11_LaoCai_GMSU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/12_BenTre_DKWF.xls.xlsx b/data/2017-old/12_BenTre_DKWF.xls.xlsx deleted file mode 100644 index 8a7cddb..0000000 Binary files a/data/2017-old/12_BenTre_DKWF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/12_LongAn_ZZUK.xls.xlsx b/data/2017-old/12_LongAn_ZZUK.xls.xlsx deleted file mode 100644 index 8f11d78..0000000 Binary files a/data/2017-old/12_LongAn_ZZUK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/13_NamDinh_ESEL.xls.xlsx b/data/2017-old/13_NamDinh_ESEL.xls.xlsx deleted file mode 100644 index b296577..0000000 Binary files a/data/2017-old/13_NamDinh_ESEL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/13_TraVinh_LKUJ.xls.xlsx b/data/2017-old/13_TraVinh_LKUJ.xls.xlsx deleted file mode 100644 index 340c20e..0000000 Binary files a/data/2017-old/13_TraVinh_LKUJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/14_NgheAn_BSLY.xls.xlsx b/data/2017-old/14_NgheAn_BSLY.xls.xlsx deleted file mode 100644 index b1f2891..0000000 Binary files a/data/2017-old/14_NgheAn_BSLY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/15_PhuTho_ABWQ.xls.xlsx b/data/2017-old/15_PhuTho_ABWQ.xls.xlsx deleted file mode 100644 index 5135dac..0000000 Binary files a/data/2017-old/15_PhuTho_ABWQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/16_QuangBinh_KGEU.xls.xlsx b/data/2017-old/16_QuangBinh_KGEU.xls.xlsx deleted file mode 100644 index c12f598..0000000 Binary files a/data/2017-old/16_QuangBinh_KGEU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/17_QuangNam_AMTK.xls.xlsx b/data/2017-old/17_QuangNam_AMTK.xls.xlsx deleted file mode 100644 index 761a8df..0000000 Binary files a/data/2017-old/17_QuangNam_AMTK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/18_QuangNgai_KOFP.xls.xlsx b/data/2017-old/18_QuangNgai_KOFP.xls.xlsx deleted file mode 100644 index 349bede..0000000 Binary files a/data/2017-old/18_QuangNgai_KOFP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/19_QuangTri_OMZF.xls.xlsx b/data/2017-old/19_QuangTri_OMZF.xls.xlsx deleted file mode 100644 index f66a8bb..0000000 Binary files a/data/2017-old/19_QuangTri_OMZF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx b/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx deleted file mode 100644 index 35be1a9..0000000 Binary files a/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx b/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx deleted file mode 100644 index 4a63447..0000000 Binary files a/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx b/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx deleted file mode 100644 index e55102c..0000000 Binary files a/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Son_La_JIDP.xls.xlsx b/data/2017-old/1_Son_La_JIDP.xls.xlsx deleted file mode 100644 index 62d7b18..0000000 Binary files a/data/2017-old/1_Son_La_JIDP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx b/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx deleted file mode 100644 index 601bb9e..0000000 Binary files a/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/20_TayNinh_ILFA.xls.xlsx b/data/2017-old/20_TayNinh_ILFA.xls.xlsx deleted file mode 100644 index 2b636fc..0000000 Binary files a/data/2017-old/20_TayNinh_ILFA.xls.xlsx and /dev/null differ diff --git a/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx b/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx deleted file mode 100644 index a9f5bae..0000000 Binary files a/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx b/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx deleted file mode 100644 index 8f9c2bb..0000000 Binary files a/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/23_HaiPhong_HXBV.xls.xlsx b/data/2017-old/23_HaiPhong_HXBV.xls.xlsx deleted file mode 100644 index c219f1f..0000000 Binary files a/data/2017-old/23_HaiPhong_HXBV.xls.xlsx and /dev/null differ diff --git a/data/2017-old/24_HCM_XULN.xls.xlsx b/data/2017-old/24_HCM_XULN.xls.xlsx deleted file mode 100644 index a526aac..0000000 Binary files a/data/2017-old/24_HCM_XULN.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_BacKan_GFVQ.xls.xlsx b/data/2017-old/2_BacKan_GFVQ.xls.xlsx deleted file mode 100644 index 115d551..0000000 Binary files a/data/2017-old/2_BacKan_GFVQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx b/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx deleted file mode 100644 index 2750bed..0000000 Binary files a/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx b/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx deleted file mode 100644 index 042f58d..0000000 Binary files a/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx b/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx deleted file mode 100644 index 351d53e..0000000 Binary files a/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx b/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx deleted file mode 100644 index cfb98f9..0000000 Binary files a/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_BacGiang_TOIF.xls.xlsx b/data/2017-old/3_BacGiang_TOIF.xls.xlsx deleted file mode 100644 index 1e9661b..0000000 Binary files a/data/2017-old/3_BacGiang_TOIF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx b/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx deleted file mode 100644 index 592d676..0000000 Binary files a/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx b/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx deleted file mode 100644 index f68c4e7..0000000 Binary files a/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx b/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx deleted file mode 100644 index 5afdc2d..0000000 Binary files a/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx b/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx deleted file mode 100644 index 34d591f..0000000 Binary files a/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_An_Giang_JNOS.xls.xlsx b/data/2017-old/4_An_Giang_JNOS.xls.xlsx deleted file mode 100644 index b4032d3..0000000 Binary files a/data/2017-old/4_An_Giang_JNOS.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_BacNinh_STLR.xls.xlsx b/data/2017-old/4_BacNinh_STLR.xls.xlsx deleted file mode 100644 index cc08868..0000000 Binary files a/data/2017-old/4_BacNinh_STLR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx b/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx deleted file mode 100644 index ddb0fc5..0000000 Binary files a/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_DienBien_SOJG.xls.xlsx b/data/2017-old/4_DienBien_SOJG.xls.xlsx deleted file mode 100644 index 0dddaac..0000000 Binary files a/data/2017-old/4_DienBien_SOJG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_Lang_Son_NYQL.xls.xlsx b/data/2017-old/4_Lang_Son_NYQL.xls.xlsx deleted file mode 100644 index a845623..0000000 Binary files a/data/2017-old/4_Lang_Son_NYQL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx b/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx deleted file mode 100644 index 73c1bef..0000000 Binary files a/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx b/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx deleted file mode 100644 index c0f9294..0000000 Binary files a/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_HaiDuong_WWWG.xls.xlsx b/data/2017-old/5_HaiDuong_WWWG.xls.xlsx deleted file mode 100644 index 190b19a..0000000 Binary files a/data/2017-old/5_HaiDuong_WWWG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Hanam_QGJS.xls.xlsx b/data/2017-old/5_Hanam_QGJS.xls.xlsx deleted file mode 100644 index 1740b8b..0000000 Binary files a/data/2017-old/5_Hanam_QGJS.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx b/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx deleted file mode 100644 index 09a6e5f..0000000 Binary files a/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx b/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx deleted file mode 100644 index 35fbd77..0000000 Binary files a/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx b/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx deleted file mode 100644 index 7543315..0000000 Binary files a/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_HoaBinh_TPZY.xls.xlsx b/data/2017-old/6_HoaBinh_TPZY.xls.xlsx deleted file mode 100644 index 932db3e..0000000 Binary files a/data/2017-old/6_HoaBinh_TPZY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_NinhBinh_IGFT.xls.xlsx b/data/2017-old/6_NinhBinh_IGFT.xls.xlsx deleted file mode 100644 index 2553671..0000000 Binary files a/data/2017-old/6_NinhBinh_IGFT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx b/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx deleted file mode 100644 index c04950f..0000000 Binary files a/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_HaTinh_DDHD.xls.xlsx b/data/2017-old/7_HaTinh_DDHD.xls.xlsx deleted file mode 100644 index 9327104..0000000 Binary files a/data/2017-old/7_HaTinh_DDHD.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_HungYen_LTIK.xls.xlsx b/data/2017-old/7_HungYen_LTIK.xls.xlsx deleted file mode 100644 index bfff635..0000000 Binary files a/data/2017-old/7_HungYen_LTIK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx b/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx deleted file mode 100644 index 3563aab..0000000 Binary files a/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx b/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx deleted file mode 100644 index 8e51dda..0000000 Binary files a/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_Can_Tho_RQZM.xls.xlsx b/data/2017-old/8_Can_Tho_RQZM.xls.xlsx deleted file mode 100644 index 172a18d..0000000 Binary files a/data/2017-old/8_Can_Tho_RQZM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx b/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx deleted file mode 100644 index 98c80a6..0000000 Binary files a/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_KienGiang_OTOB.xls.xlsx b/data/2017-old/8_KienGiang_OTOB.xls.xlsx deleted file mode 100644 index fa07efa..0000000 Binary files a/data/2017-old/8_KienGiang_OTOB.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_PhuYen_OGIM.xls.xlsx b/data/2017-old/8_PhuYen_OGIM.xls.xlsx deleted file mode 100644 index 02e391c..0000000 Binary files a/data/2017-old/8_PhuYen_OGIM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_DakNong_FBOP.xls.xlsx b/data/2017-old/9_DakNong_FBOP.xls.xlsx deleted file mode 100644 index 6ae062c..0000000 Binary files a/data/2017-old/9_DakNong_FBOP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx b/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx deleted file mode 100644 index d30f3f7..0000000 Binary files a/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_LaiChau_ALKN.xls.xlsx b/data/2017-old/9_LaiChau_ALKN.xls.xlsx deleted file mode 100644 index 9806b8e..0000000 Binary files a/data/2017-old/9_LaiChau_ALKN.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx b/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx deleted file mode 100644 index 7ba2885..0000000 Binary files a/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx and /dev/null differ diff --git a/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx b/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx deleted file mode 100644 index 3a26735..0000000 Binary files a/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx and /dev/null differ diff --git a/data/2017-old2/1.Da Nang_ABWU.xlsx b/data/2017-old2/1.Da Nang_ABWU.xlsx deleted file mode 100644 index 5396aef..0000000 Binary files a/data/2017-old2/1.Da Nang_ABWU.xlsx and /dev/null differ diff --git a/data/2017-old2/1.Son La_XLFN.xlsx b/data/2017-old2/1.Son La_XLFN.xlsx deleted file mode 100644 index 7eafde3..0000000 Binary files a/data/2017-old2/1.Son La_XLFN.xlsx and /dev/null differ diff --git a/data/2017-old2/1.TuyenQuang_PTMR.xlsx b/data/2017-old2/1.TuyenQuang_PTMR.xlsx deleted file mode 100644 index 06f7c27..0000000 Binary files a/data/2017-old2/1.TuyenQuang_PTMR.xlsx and /dev/null differ diff --git a/data/2017-old2/10.BinhThuan_MVVG.xlsx b/data/2017-old2/10.BinhThuan_MVVG.xlsx deleted file mode 100644 index 8cabd6d..0000000 Binary files a/data/2017-old2/10.BinhThuan_MVVG.xlsx and /dev/null differ diff --git a/data/2017-old2/10.LamDong_YUQA.xlsx b/data/2017-old2/10.LamDong_YUQA.xlsx deleted file mode 100644 index e4debd8..0000000 Binary files a/data/2017-old2/10.LamDong_YUQA.xlsx and /dev/null differ diff --git a/data/2017-old2/10.Soc Trang_LQWU.xlsx b/data/2017-old2/10.Soc Trang_LQWU.xlsx deleted file mode 100644 index c8cd707..0000000 Binary files a/data/2017-old2/10.Soc Trang_LQWU.xlsx and /dev/null differ diff --git a/data/2017-old2/11.BinhDuong_HVAH.xlsx b/data/2017-old2/11.BinhDuong_HVAH.xlsx deleted file mode 100644 index 8b3f7ad..0000000 Binary files a/data/2017-old2/11.BinhDuong_HVAH.xlsx and /dev/null differ diff --git a/data/2017-old2/11.LaoCai_ZTBP.xlsx b/data/2017-old2/11.LaoCai_ZTBP.xlsx deleted file mode 100644 index 63870d1..0000000 Binary files a/data/2017-old2/11.LaoCai_ZTBP.xlsx and /dev/null differ diff --git a/data/2017-old2/12.BenTre_NQTU.xlsx b/data/2017-old2/12.BenTre_NQTU.xlsx deleted file mode 100644 index ae9edb4..0000000 Binary files a/data/2017-old2/12.BenTre_NQTU.xlsx and /dev/null differ diff --git a/data/2017-old2/12.LongAn_PDRH.xlsx b/data/2017-old2/12.LongAn_PDRH.xlsx deleted file mode 100644 index e0b66c8..0000000 Binary files a/data/2017-old2/12.LongAn_PDRH.xlsx and /dev/null differ diff --git a/data/2017-old2/13.NamDinh_NAYR.xlsx b/data/2017-old2/13.NamDinh_NAYR.xlsx deleted file mode 100644 index b16ead2..0000000 Binary files a/data/2017-old2/13.NamDinh_NAYR.xlsx and /dev/null differ diff --git a/data/2017-old2/13.TraVinh_FODZ.xlsx b/data/2017-old2/13.TraVinh_FODZ.xlsx deleted file mode 100644 index eb2c6ce..0000000 Binary files a/data/2017-old2/13.TraVinh_FODZ.xlsx and /dev/null differ diff --git a/data/2017-old2/14.NgheAn_HTKD.xlsx b/data/2017-old2/14.NgheAn_HTKD.xlsx deleted file mode 100644 index f872da1..0000000 Binary files a/data/2017-old2/14.NgheAn_HTKD.xlsx and /dev/null differ diff --git a/data/2017-old2/15.PhuTho_IJZW.xlsx b/data/2017-old2/15.PhuTho_IJZW.xlsx deleted file mode 100644 index e075d3a..0000000 Binary files a/data/2017-old2/15.PhuTho_IJZW.xlsx and /dev/null differ diff --git a/data/2017-old2/17.QuangNam_NQMG.xlsx b/data/2017-old2/17.QuangNam_NQMG.xlsx deleted file mode 100644 index 3e75262..0000000 Binary files a/data/2017-old2/17.QuangNam_NQMG.xlsx and /dev/null differ diff --git a/data/2017-old2/18.QuangNgai_IUPY.xlsx b/data/2017-old2/18.QuangNgai_IUPY.xlsx deleted file mode 100644 index 5e37ea2..0000000 Binary files a/data/2017-old2/18.QuangNgai_IUPY.xlsx and /dev/null differ diff --git a/data/2017-old2/19.QuangTri_MKNN.xlsx b/data/2017-old2/19.QuangTri_MKNN.xlsx deleted file mode 100644 index 454256c..0000000 Binary files a/data/2017-old2/19.QuangTri_MKNN.xlsx and /dev/null differ diff --git a/data/2017-old2/2.BacKan_YQNX.xlsx b/data/2017-old2/2.BacKan_YQNX.xlsx deleted file mode 100644 index 6fa41c9..0000000 Binary files a/data/2017-old2/2.BacKan_YQNX.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Ha Giang_PIYK.xlsx b/data/2017-old2/2.Ha Giang_PIYK.xlsx deleted file mode 100644 index d65b471..0000000 Binary files a/data/2017-old2/2.Ha Giang_PIYK.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Ninh Thuan_BAGG.xlsx b/data/2017-old2/2.Ninh Thuan_BAGG.xlsx deleted file mode 100644 index a77706b..0000000 Binary files a/data/2017-old2/2.Ninh Thuan_BAGG.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Thanh Hoa_UOPE.xlsx b/data/2017-old2/2.Thanh Hoa_UOPE.xlsx deleted file mode 100644 index d40be28..0000000 Binary files a/data/2017-old2/2.Thanh Hoa_UOPE.xlsx and /dev/null differ diff --git a/data/2017-old2/2.VinhPhuc_QZJK.xlsx b/data/2017-old2/2.VinhPhuc_QZJK.xlsx deleted file mode 100644 index be86a48..0000000 Binary files a/data/2017-old2/2.VinhPhuc_QZJK.xlsx and /dev/null differ diff --git a/data/2017-old2/20.TayNinh_KJAQ.xlsx b/data/2017-old2/20.TayNinh_KJAQ.xlsx deleted file mode 100644 index 65eedff..0000000 Binary files a/data/2017-old2/20.TayNinh_KJAQ.xlsx and /dev/null differ diff --git a/data/2017-old2/21.ThaiBinh_KTQN.xlsx b/data/2017-old2/21.ThaiBinh_KTQN.xlsx deleted file mode 100644 index 02a3895..0000000 Binary files a/data/2017-old2/21.ThaiBinh_KTQN.xlsx and /dev/null differ diff --git a/data/2017-old2/22.ThaiNguyen_BKIF.xlsx b/data/2017-old2/22.ThaiNguyen_BKIF.xlsx deleted file mode 100644 index a144e9c..0000000 Binary files a/data/2017-old2/22.ThaiNguyen_BKIF.xlsx and /dev/null differ diff --git a/data/2017-old2/24.HCM_UTLQ.xlsx b/data/2017-old2/24.HCM_UTLQ.xlsx deleted file mode 100644 index cc42e51..0000000 Binary files a/data/2017-old2/24.HCM_UTLQ.xlsx and /dev/null differ diff --git a/data/2017-old2/3.BacGiang_SAVS.xlsx b/data/2017-old2/3.BacGiang_SAVS.xlsx deleted file mode 100644 index bc9b5f6..0000000 Binary files a/data/2017-old2/3.BacGiang_SAVS.xlsx and /dev/null differ diff --git a/data/2017-old2/3.BinhPhuoc_IPHL.xlsx b/data/2017-old2/3.BinhPhuoc_IPHL.xlsx deleted file mode 100644 index 8c5871d..0000000 Binary files a/data/2017-old2/3.BinhPhuoc_IPHL.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Cao Bang_WMUU.xlsx b/data/2017-old2/3.Cao Bang_WMUU.xlsx deleted file mode 100644 index 36200fb..0000000 Binary files a/data/2017-old2/3.Cao Bang_WMUU.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Dong Thap_HKJX.xlsx b/data/2017-old2/3.Dong Thap_HKJX.xlsx deleted file mode 100644 index ae37166..0000000 Binary files a/data/2017-old2/3.Dong Thap_HKJX.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx b/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx deleted file mode 100644 index 587af78..0000000 Binary files a/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx and /dev/null differ diff --git a/data/2017-old2/4.An Giang_PMJD.xlsx b/data/2017-old2/4.An Giang_PMJD.xlsx deleted file mode 100644 index b651ffb..0000000 Binary files a/data/2017-old2/4.An Giang_PMJD.xlsx and /dev/null differ diff --git a/data/2017-old2/4.BacNinh_NNIS.xlsx b/data/2017-old2/4.BacNinh_NNIS.xlsx deleted file mode 100644 index 04de119..0000000 Binary files a/data/2017-old2/4.BacNinh_NNIS.xlsx and /dev/null differ diff --git a/data/2017-old2/4.Binh Dinh_VOMJ.xlsx b/data/2017-old2/4.Binh Dinh_VOMJ.xlsx deleted file mode 100644 index 704c4ae..0000000 Binary files a/data/2017-old2/4.Binh Dinh_VOMJ.xlsx and /dev/null differ diff --git a/data/2017-old2/4.DienBien_FYGN.xlsx b/data/2017-old2/4.DienBien_FYGN.xlsx deleted file mode 100644 index aae73eb..0000000 Binary files a/data/2017-old2/4.DienBien_FYGN.xlsx and /dev/null differ diff --git a/data/2017-old2/4.Lang Son_QWOG.xlsx b/data/2017-old2/4.Lang Son_QWOG.xlsx deleted file mode 100644 index e7e9534..0000000 Binary files a/data/2017-old2/4.Lang Son_QWOG.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Bac Lieu_VIVY.xlsx b/data/2017-old2/5.Bac Lieu_VIVY.xlsx deleted file mode 100644 index 575cac0..0000000 Binary files a/data/2017-old2/5.Bac Lieu_VIVY.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Gia Lai_TAAS.xlsx b/data/2017-old2/5.Gia Lai_TAAS.xlsx deleted file mode 100644 index f358c30..0000000 Binary files a/data/2017-old2/5.Gia Lai_TAAS.xlsx and /dev/null differ diff --git a/data/2017-old2/5.HaiDuong_WNHD.xlsx b/data/2017-old2/5.HaiDuong_WNHD.xlsx deleted file mode 100644 index 7b1169f..0000000 Binary files a/data/2017-old2/5.HaiDuong_WNHD.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Hanam_SDKN.xlsx b/data/2017-old2/5.Hanam_SDKN.xlsx deleted file mode 100644 index 5b9d113..0000000 Binary files a/data/2017-old2/5.Hanam_SDKN.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Yen Bai_BSLV.xlsx b/data/2017-old2/5.Yen Bai_BSLV.xlsx deleted file mode 100644 index 915dc38..0000000 Binary files a/data/2017-old2/5.Yen Bai_BSLV.xlsx and /dev/null differ diff --git a/data/2017-old2/6.Hau Giang_SIAJ.xlsx b/data/2017-old2/6.Hau Giang_SIAJ.xlsx deleted file mode 100644 index f4a67e4..0000000 Binary files a/data/2017-old2/6.Hau Giang_SIAJ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.HoaBinh_HLYQ.xlsx b/data/2017-old2/6.HoaBinh_HLYQ.xlsx deleted file mode 100644 index 768a885..0000000 Binary files a/data/2017-old2/6.HoaBinh_HLYQ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.NinhBinh_PKMQ.xlsx b/data/2017-old2/6.NinhBinh_PKMQ.xlsx deleted file mode 100644 index 7fab324..0000000 Binary files a/data/2017-old2/6.NinhBinh_PKMQ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.Quang Ninh_YAKJ.xlsx b/data/2017-old2/6.Quang Ninh_YAKJ.xlsx deleted file mode 100644 index 51ddb21..0000000 Binary files a/data/2017-old2/6.Quang Ninh_YAKJ.xlsx and /dev/null differ diff --git a/data/2017-old2/7.HaTinh_XMFQ.xlsx b/data/2017-old2/7.HaTinh_XMFQ.xlsx deleted file mode 100644 index 0aa1629..0000000 Binary files a/data/2017-old2/7.HaTinh_XMFQ.xlsx and /dev/null differ diff --git a/data/2017-old2/7.HungYen_TCBE.xlsx b/data/2017-old2/7.HungYen_TCBE.xlsx deleted file mode 100644 index be95a39..0000000 Binary files a/data/2017-old2/7.HungYen_TCBE.xlsx and /dev/null differ diff --git a/data/2017-old2/7.Kon Tum_CAQU.xlsx b/data/2017-old2/7.Kon Tum_CAQU.xlsx deleted file mode 100644 index aa2d88a..0000000 Binary files a/data/2017-old2/7.Kon Tum_CAQU.xlsx and /dev/null differ diff --git a/data/2017-old2/7.Tien Giang_AOWZ.xlsx b/data/2017-old2/7.Tien Giang_AOWZ.xlsx deleted file mode 100644 index 43206a0..0000000 Binary files a/data/2017-old2/7.Tien Giang_AOWZ.xlsx and /dev/null differ diff --git a/data/2017-old2/8.PhuYen_MLTQ.xlsx b/data/2017-old2/8.PhuYen_MLTQ.xlsx deleted file mode 100644 index 7af0186..0000000 Binary files a/data/2017-old2/8.PhuYen_MLTQ.xlsx and /dev/null differ diff --git a/data/2017-old2/9.DakNong_ZWJT.xlsx b/data/2017-old2/9.DakNong_ZWJT.xlsx deleted file mode 100644 index cffd8a6..0000000 Binary files a/data/2017-old2/9.DakNong_ZWJT.xlsx and /dev/null differ diff --git a/data/2017-old2/9.Khanh Hoa_IBTX.xlsx b/data/2017-old2/9.Khanh Hoa_IBTX.xlsx deleted file mode 100644 index 058419d..0000000 Binary files a/data/2017-old2/9.Khanh Hoa_IBTX.xlsx and /dev/null differ diff --git a/data/2017-old2/9.Vinh Long_WVKI.xlsx b/data/2017-old2/9.Vinh Long_WVKI.xlsx deleted file mode 100644 index febfad4..0000000 Binary files a/data/2017-old2/9.Vinh Long_WVKI.xlsx and /dev/null differ diff --git a/datasets.json b/datasets.json new file mode 100644 index 0000000..aab7660 --- /dev/null +++ b/datasets.json @@ -0,0 +1,31 @@ +{ + "_comment": [ + "The dataset registry: the one place every stage agrees on what exists.", + "", + " crawler/ fills data//", + " parser/ reads data// with configs/.yml, writes .db", + " assembler/ verifies, compresses and publishes it as db/.db.gz", + " web/ serves it at /thptqg//", + "", + "It is JSON rather than a module because Go and the Vite app both read it,", + "and JSON is the only format both parse without a dependency. Presentation", + "(titles, labels, SQL presets) stays in web/src/datasets.js keyed by id;", + "that file fails loudly if the two lists disagree.", + "", + "expectedRows is exact, not approximate. The inputs are frozen historical", + "exam results, so a deviation of even one row means something changed that", + "nobody intended, and the assembler refuses to publish." + ], + "datasets": [ + { + "id": "2016", + "expectedRows": 877461, + "dbSizeMb": 44 + }, + { + "id": "2017", + "expectedRows": 861068, + "dbSizeMb": 48 + } + ] +} diff --git a/docs/README.md b/docs/README.md index b2ad89a..0c9ef0a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Docs -- [`project-overview.md`](./project-overview.md) — goal, scope, constraints, the four datasets, history +- [`project-overview.md`](./project-overview.md) — goal, scope, constraints, the datasets, history - [`system-architecture.md`](./system-architecture.md) — data flow, canonical schema, routing, how one frontend serves both exam years - [`data-pipeline.md`](./data-pipeline.md) — Excel parse quirks, per-dataset formats, overflow-sheet gotcha, expected row counts, verifying a rebuild - [`deployment-guide.md`](./deployment-guide.md) — GitHub Pages workflow, adding a dataset, rollback, troubleshooting diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index aa8d477..4ee55db 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -2,36 +2,85 @@ From raw Excel files to a compressed SQLite file the browser can load. -One Rust binary (`parser/`) builds every dataset. What differs per dataset is +One Go binary (`parser/`) builds every dataset. What differs per dataset is parse rules only — sheet strategy, column layout, validation guards — declared -in `parser/configs/.toml`. The table shape, the INSERT and the subject -regexes are canonical and live in `parser/src/schema.rs`. +in `parser/configs/.yml`. The table shape, the INSERT and the subject +regexes are canonical and live in `parser/internal/schema/schema.go`. ## Sources -| id | Files | Reproducible | Origin | +| id | Files | Origin | Host live? | | --- | --- | --- | --- | -| `2016` | 4 `.xls` + 115 `.xlsx` | no | Bộ GD&ĐT, collected 2016 | -| `2017` | 63 `.xls` | **yes** | baotintuc.vn CDN | -| `2017-old` | 63 `.xlsx` | no | pre-refresh archive | -| `2017-old2` | 54 `.xlsx` | no | corrected re-export | +| `2016` | 4 `.xls` + 115 `.xlsx` | aggregator article, 119 exam clusters | unconfirmed | +| `2017` | 63 `.xls` | baotintuc.vn CDN | **yes** | -Only `2017` can be re-fetched: +Crawling lives in `crawler/`, a separate Go module. It is never part of the +build — the source files are committed, so a crawl only refreshes them. Both +runs are idempotent: files already present are skipped. ```bash -node parser/scripts/crawl-baotintuc.js +go -C crawler run ./cmd/crawl 2016 # sources/source_2016.go +go -C crawler run ./cmd/crawl 2017 # sources/source_2017.go +go -C crawler run ./cmd/crawl 2017 --list # list only, download nothing ``` -Idempotent — skips files already present, saves to `data/2017/`. Source article: +**2017** comes from the article `https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm` +and its CDN is still serving the files. -The other three were collected from Vietnamese news sites at the time and the -publisher URLs were not recorded. **The files committed in git are the only -copy.** +**2016** comes from the aggregator article +`cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html`, served from +a mirror — the site that first published it (`dtntbacgiang.edu.vn`) no longer +resolves. + +That mirror is **not reachable from every network.** It resolves to a Vietnamese +address that times out from at least some hosts abroad, in which case the crawl +stops with a connection error before downloading anything. `data/2016/` is +therefore still the only confirmed copy: do not delete it on the assumption that +a crawl can restore it. + +## How a source is defined + +A source carries no link list. It names the article that published the files and +says how to name what it finds there: + +| field | meaning | +| --- | --- | +| `Article` | the page to read links from | +| `Exts` | which file extensions to pick out of it | +| `WantFiles` | how many links to expect — fewer aborts the crawl | +| `Dest` | the local filename for one link | + +`internal/article` does the fetching and HTML parsing; `internal/fetch` does the +downloading. Because `Article` is read at run time, `--list` needs network access +too. + +`WantFiles` exists because a partial crawl is otherwise silent: parser will +build a short database from whatever files are present, and only the row-count +guard would notice, after the fact. A page that changes shape stops the crawl +instead. + +### Local filenames are load-bearing + +`parser` sorts its input files and inserts with `INSERT OR REPLACE`, which is +last-wins, so **filenames decide which row survives a duplicate exam number**. A +re-crawl that names files differently can produce a database with the same row +count and different content, which the assembler's row-count guard would +not catch. + +Each source therefore pins its local names, and +`crawler/internal/sources/sources_test.go` checks every source against its +committed `data//` in both directions — every name it would write exists, +and every file present is accounted for. + +2017 derives its names from the province name in the link text, transliterated +to ASCII (the CDN's own names are inconsistent: +`Angiang.xls`, `1BaRiaVungTau.xls`, `23HaiPhong.xls`). 2016 keeps the +server-assigned names verbatim, since it did not choose them. ## Source Excel shapes -The 2017 datasets share one layout: +2017 has one layout: | Col | Name | Content | | --- | --- | --- | @@ -41,7 +90,7 @@ The 2017 datasets share one layout: | 3 | DIEM_THI | concatenated per-subject scores, e.g. `"Toán: 6.80 Ngữ văn: 5.25 …"` | 2016 has **three** layouts across its 119 files, chosen per file at runtime via -`format_detection = "thptqg2016"` in its config: +`format_detection: thptqg2016` in its config: | Format | Detected by | Notes | | --- | --- | --- | @@ -54,7 +103,7 @@ which is why only 2016 populates those columns. ## Score text parsing -`SCORE_PATTERNS` in `parser/src/schema.rs` defines one regex per subject, and +`SCORE_PATTERNS` in `parser/internal/schema/schema.go` defines one regex per subject, and **all 16 run against every dataset**. A subject a given exam year did not offer simply never matches and stays NULL. @@ -95,8 +144,6 @@ scores rather than false matches. | --- | --- | --- | | `2016` | all sheets | per-file format detection; header-token rows rejected | | `2017` | all sheets — Hà Nội and HCM overflow | none | -| `2017-old` | first sheet only | reject non-numeric SBD (rejects a header leak in this export) | -| `2017-old2` | all sheets — HCM overflows | reject non-numeric SBD; skip blank rows before counting | ### Overflow-sheet gotcha @@ -110,40 +157,61 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what | id | Source rows | Skipped | DB rows | | --- | --- | --- | --- | | `2016` | 877,464 | 3 duplicate SBDs collapsed | **877,461** | -| `2017` | 861,131 | 63 empty | **861,068** | -| `2017-old` | 847,349 | 1 header leak | **847,348** | -| `2017-old2` | 679,764 | 0 | **679,764** | +| `2017` | 861,068 | 0 | **861,068** | ## Verifying a rebuild -`parser/scripts/db-stats.js` dumps row counts, per-column non-NULL counts, file -size and a deterministic student sample as JSON. -`parser/scripts/verify-parity.js` diffs two such files and exits non-zero on any -mismatch. +The assembler verifies itself: each database's row count must match the +figure in the table above, and each `.db.gz` must be at least 90% of its usual +size, or the build fails rather than publishing. That guard is the reason a +truncated dataset cannot reach the site with a green pipeline. + +For a deeper check, `parser/scripts/differential-parity.mjs` compares two sets +of databases field-by-field — row counts, per-column non-NULL counts, a +full-table SHA-256 over every row ordered by `so_bao_danh`, schema metadata, and +build stdout: ```bash -node parser/scripts/db-stats.js 2016=.db … > current.json -node parser/scripts/verify-parity.js plans/reports/parser-parity-baseline.json current.json +node parser/scripts/differential-parity.mjs \ + --rust /path/to/a-{id}.db --go /path/to/b-{id}.db ``` -The committed baseline was built with the pre-refactor code and cannot be -regenerated — the two old crates no longer exist. Both scripts use the built-in -`node:sqlite`, so they need no dependencies. +It exits non-zero on any mismatch and fails loudly if a dataset is missing rather +than skipping it. Written for the Rust-to-Go migration, it works for any two +builds. Uses the built-in `node:sqlite`, so it needs no dependencies. + +`parser/internal/reader` additionally carries a frozen oracle of per-file +cell-dump hashes covering all 182 inputs; `go -C parser test ./...` fails if any single +cell of any input file reads differently. ## Refreshing the 2017 data ```bash rm data/2017/*.xls -node parser/scripts/crawl-baotintuc.js -node parser/scripts/build-db.js 2017 +go -C crawler run ./cmd/crawl 2017 +go -C assembler run ./cmd/assemble db 2017 ``` -Then re-run the parity check above and confirm the row count still matches. +The row-count guard in `build:db` confirms the rebuild matches the expected +total. That guard checks the count only, so if the crawl was expected to change +the data, compare content with `differential-parity.mjs` against a copy of the +previous database rather than trusting the count. -## Legacy scripts +## Removed scripts -`parser/scripts/check-duplicates.js` and `diff-datasets.js` are one-off audits -that were already broken before the repo was unified — a hardcoded Windows path -in one, an undeclared `better-sqlite3` dependency and stale paths in the other. -Each carries a comment saying so. For comparing two builds, use `db-stats.js` -plus `verify-parity.js` instead. +`check-duplicates.js`, `diff-datasets.js`, `db-stats.js` and `verify-parity.js` +were dropped with the Rust parser. The first two had been broken since before the +repo was unified (a hardcoded Windows path in one, an undeclared +`better-sqlite3` dependency in the other) and neither had any automated caller. +The latter two are superseded by `differential-parity.mjs`, which compares more +and cannot silently skip a dataset. + +`crawl-baotintuc.js` was not dropped but rewritten as the Go `crawler/` module, +producing the same local filenames. Two changes beyond the port: + +- It carried its 63 links as a hardcoded array. The Go version reads them from + the article instead, so the list cannot drift from what was published. +- Downloads land on a `.part` file and are renamed on completion. Writing + straight to the destination left a truncated file after an interrupted run, + and since the skip check only tests for a non-empty file, every later run + would skip it — the corruption was permanent and silent. diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index f5b29b8..3f4cf4d 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -1,20 +1,20 @@ # Deployment Guide Deploys to GitHub Pages via `.github/workflows/deploy-pages.yml`. Every push to -`main` rebuilds all four datasets and redeploys the whole site. +`main` rebuilds both datasets and redeploys the whole site. One-time setup: **Settings → Pages → Source: GitHub Actions**. ## What the workflow does -1. Checkout, Rust toolchain, Node 24, `npm ci` -2. `npm run build:rust` — one parser binary -3. `npm run build:db` — builds and gzips all four databases into - `.build/public/db/` -4. `npm run build:site` — one Vite build, then `scripts/assemble-site.js` -5. `actions/upload-pages-artifact` + `actions/deploy-pages` +1. Checkout, Go toolchain, Node 24, `npm ci` in `web/` +2. Parser and crawler test suites, web lint, `govulncheck` over all three modules +3. `go -C assembler run ./cmd/assemble` — the whole pipeline: compile the + parser, build and verify each database, compress it into `.build/public/db/`, + run the Vite build, assemble `_site/` +4. `actions/upload-pages-artifact` + `actions/deploy-pages` -The database build dominates the runtime: roughly 419 MB of Excel is parsed on +The database build dominates the runtime: roughly 348 MB of Excel is parsed on every deploy. ## Resulting URLs @@ -23,28 +23,25 @@ every deploy. https://.github.io/thptqg/ https://.github.io/thptqg/2016/ https://.github.io/thptqg/2017/ -https://.github.io/thptqg/2017-old/ -https://.github.io/thptqg/2017-old2/ ``` -`/thptqg/2017/old/` and `/thptqg/2017/old2/` were the pre-flattening URLs. They -are still served, and the router rewrites them to the flat form with the query -string intact. +`/thptqg/2017/old/` and `/thptqg/2017/old2/` were the pre-flattening URLs for +the two removed 2017 archives. They are no longer served; like any unknown path +they now render the hub via `404.html`. ## Local reproduction ```bash -npm ci -npm run build:rust -npm run build:db # all four; pass an id to build just one -npm run build:site # vite build + assemble into _site/ +(cd web && npm ci) +go -C assembler run ./cmd/assemble npx serve _site ``` -To rebuild a single dataset: +To rebuild a single dataset, or only the site: ```bash -node parser/scripts/build-db.js 2017-old +go -C assembler run ./cmd/assemble db 2017 +go -C assembler run ./cmd/assemble site ``` ## Base path @@ -56,19 +53,21 @@ up as a blank page with 404s on `/assets/...`. ## Adding a dataset 1. Put the Excel files in `data//` -2. Add `parser/configs/.toml` with the parse rules — sheet mode, column +2. Add `parser/configs/.yml` with the parse rules — sheet mode, column indices, SBD validation, header tokens, blank-row stripping. No SQL: the - schema is canonical and lives in `parser/src/schema.rs` -3. Add an entry to `DATASETS` in `src/datasets.js` + schema is canonical and lives in `parser/internal/schema/schema.go` +3. Add an entry to `datasets.json` — id, `expectedRows`, `dbSizeMb` +4. Add its presentation to `CONTENT` in `web/src/datasets.js` -Nothing else. The build script, the site assembly and the router all read that -one list, and the frontend adapts to whichever columns the dataset populates. +Nothing else. The assembler and the router both read the registry, and the +frontend adapts to whichever columns the dataset populates. The last two steps +check each other, so forgetting either fails rather than half-working. ## Why no uncompressed database can ship -`build-db.js` runs `gzip -9` **without** `-k`, so the raw file does not survive -the build. `assemble-site.js` then fails the job if any `.db`, `.db-journal`, -`.db-wal` or `.db-shm` reached the output. +The assembler deletes the source once compression succeeds, so the raw file +does not survive the build, and it then fails the job if any `.db`, +`.db-journal`, `.db-wal` or `.db-shm` reached the output. Both guards exist because the previous pipeline wrote a 100+ MB uncompressed database into the source tree and relied on an `rm` step to keep it out of the @@ -98,8 +97,8 @@ run rebuilds the older state. There is no data to migrate. | Symptom | Typical cause | | --- | --- | | Blank page, 404 on assets | `base` in `vite.config.js` does not match the repo name | -| `Failed to fetch database: 404` | Dataset id in `src/datasets.js` does not match the file in `db/` | -| A route 404s | `assemble-site.js` did not run, or the id is missing from `DATASETS` | +| `Failed to fetch database: 404` | Dataset id in `datasets.json` does not match the file in `db/` | +| A route 404s | The site step did not run, or the id is missing from `datasets.json` | | WASM fails to load | `sql.js.org` unreachable — self-host `sql-wasm.wasm` and update `SQL_WASM_URL` in `use-sqlite.js` | | Deploy fails on assembly | An uncompressed database artefact reached the output; the error names the files | | Missing rows after a data update | Unknown Excel header — check the per-file row counts the parser prints | diff --git a/docs/project-overview.md b/docs/project-overview.md index af8dfdd..7b1cba5 100644 --- a/docs/project-overview.md +++ b/docs/project-overview.md @@ -4,7 +4,7 @@ A public lookup tool for Vietnam's National High School Graduation Exam scores, running entirely in the browser and hosted for free on GitHub Pages. Covers the -2016 and 2017 exams — 1.7 million candidates across four datasets. +2016 and 2017 exams — 1.7 million candidates across two datasets. ## Scope @@ -21,7 +21,7 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the ## Constraints -- **Zero backend.** The full database (38–48 MB gzipped per dataset) is +- **Zero backend.** The full database (44–48 MB gzipped per dataset) is downloaded to the browser and queried in-process. - **Read-only.** `INSERT`/`UPDATE`/`DELETE` are rejected, so nobody is misled into thinking edits persist. `sql.js` is in-memory anyway. @@ -36,16 +36,17 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | 119 files, three column layouts | | `2017` | 2017 | 861,068 | current generation, reproducible from source | -| `2017-old` | 2017 | 847,348 | pre-refresh archive | -| `2017-old2` | 2017 | 679,764 | corrected re-export | -The three 2017 datasets are kept side by side because they disagree, and the -disagreement is itself informative. Only `2017` is re-fetchable; the rest exist -solely as the copies committed here. +Two further 2017 datasets (`2017-old`, `2017-old2`) were kept alongside these +because the three publications disagreed. They have been removed; git history +still has them. -Original 2016 aggregator link -() -is no longer accessible, which is why the raw files are mirrored in `data/2016/`. +Both datasets now have a crawler source. 2017 comes from the baotintuc.vn CDN, +which is still live. 2016 comes from an aggregator article whose original host +(`dtntbacgiang.edu.vn`) no longer resolves — its link list was recovered from +the Internet Archive and is pointed at a mirror that is still online. See +[data-pipeline](./data-pipeline.md#sources) for what that does and does not +guarantee. ## History @@ -53,7 +54,7 @@ Each year began as a standalone repository (`thptqg2016`, `thptqg2017`), merged here with full history. They initially kept separate frontends and separate copies of the same Rust parser, synchronised by hand. That duplication was removed: there is now one frontend, one parser, and one canonical schema, with -per-dataset differences confined to four small config files and one registry +per-dataset differences confined to one small config file and one registry entry each. The unification also fixed a latent data-loss bug — neither year's parser diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 7eb5cce..5de66fb 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -3,23 +3,27 @@ Static site, no backend. The browser downloads a compressed SQLite file at boot and every query runs locally via `sql.js` (SQLite compiled to WebAssembly). -One frontend, one parser, one schema, four datasets. +One frontend, one parser, one schema, two datasets. ## Data flow +Each stage is a directory; `data/` and `_site/` are the stores they hand work +through. `assembler/` sequences everything from the parser onwards. + ``` + ▲ crawler/ (Go — manual refresh only, never part of the build) data//*.xls(x) │ - ▼ parser/ (Rust, one binary, one config per dataset) + ▼ parser/ (Go, one binary, one config per dataset) .build/public/db/.db │ - ▼ gzip -9 (no -k: the raw file does not survive) - .build/public/db/.db.gz + ▼ assembler/ — row count must match datasets.json, then gzip + .build/public/db/.db.gz (the raw .db does not survive) │ - ▼ vite build (publicDir = .build/public) - dist/ + ▼ assembler/ → vite build (root = web/, publicDir = .build/public) + web/dist/ │ - ▼ scripts/assemble-site.js + ▼ assembler/ — one index.html per dataset; every database must be present _site/ → GitHub Pages │ ▼ browser @@ -31,26 +35,31 @@ data//*.xls(x) One identifier ties the whole pipeline together: ``` -data/2017-old/ → parser/configs/2017-old.toml → db/2017-old.db.gz → /thptqg/2017-old/ +data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ ``` -`src/datasets.js` declares the four ids once. The frontend, the database build -(`parser/scripts/build-db.js`) and the site assembly all import that list, so -adding a dataset means adding one entry and one config file. +`datasets.json` at the repository root declares the ids once, with the row count +and artifact size the assembler enforces. It is JSON rather than a module +because the assembler is a Go program and the Vite app is not, and JSON is the +only format both parse without a dependency. + +Presentation — titles, labels, search examples, SQL presets — stays in +`web/src/datasets.js`, keyed by id. That file cross-checks the two: a registry +entry with no content, or content for a dataset that was never built, throws at +module load rather than rendering a page with no title or a link to a database +that does not exist. | id | Exam | Rows | Source | | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | Bộ GD&ĐT | | `2017` | 2017 | 861,068 | baotintuc.vn | -| `2017-old` | 2017 | 847,348 | pre-refresh archive | -| `2017-old2` | 2017 | 679,764 | corrected re-export | ## Canonical schema -Defined once in `parser/src/schema.rs` — DDL, INSERT, column order and the 16 -subject regexes. The four TOML configs carry no SQL at all, only per-dataset -parse rules. Config parsing uses `deny_unknown_fields`, so a leftover `[schema]` -block fails loudly instead of looking effective while `schema.rs` drives the +Defined once in `parser/internal/schema/schema.go` — DDL, INSERT, column order and the 16 +subject regexes. The two YAML configs carry no SQL at all, only per-dataset +parse rules. Config parsing sets `KnownFields(true)`, so a leftover `schema:` +block fails loudly instead of looking effective while `schema.go` drives the build. ```sql @@ -73,7 +82,7 @@ CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT N Every dataset gets all 22 columns; ones it has no data for are NULL, costing about a byte per row. `khtn`, `khxh` and `gdcd` are empty on 2016; -`ten_cum_thi` and `gioi_tinh` are empty on the 2017 datasets. +`ten_cum_thi` and `gioi_tinh` are empty on 2017. `idx_ten_cum_thi` is partial, so it holds zero entries where the column is always NULL. @@ -86,15 +95,13 @@ URLs are flat, one segment per dataset, and the segment is the id: /thptqg/ hub /thptqg/2016/ /thptqg/2017/ -/thptqg/2017-old/ -/thptqg/2017-old2/ ``` -`src/router.js` is an exact match on that segment. The nested form used before -(`/thptqg/2017/old/`) would have needed longest-prefix matching, since it also -starts with `/thptqg/2017/`. Those two legacy URLs still resolve: the router -rewrites them to the flat equivalent with `history.replaceState`, preserving the -query string so `?q=` deep links survive. +`web/src/router.js` is an exact match on that segment. The nested form used +before (`/thptqg/2017/old/`) would have needed longest-prefix matching, since it +also starts with `/thptqg/2017/`. Both of those URLs addressed the two removed +2017 archives, so the rewrite that kept them working has gone with them; they +fall through to the hub like any other unknown path. A single Vite build emits one `index.html`. Because `base` is absolute (`/thptqg/`), that file references `/thptqg/assets/...` regardless of the @@ -115,11 +122,11 @@ No component contains a per-dataset conditional. Two mechanisms do the work: self-exclude wherever those languages were not sat. Anything genuinely per-dataset — title, source, database size, search examples, -SQL presets — lives in `src/datasets.js`. +SQL presets — lives in `web/src/datasets.js`. ## Exam ID formats -`src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is +`web/src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is shared by `App.jsx` and `search-form.jsx` (they previously held separate copies and had drifted apart on exactly this rule). @@ -135,7 +142,7 @@ candidates (70.3%). Letter prefixes are upper-cased before lookup, so ## Score tiers -Six-level ladder in `scoreTier()` (`src/lib/admission-blocks.js`), paired with a +Six-level ladder in `scoreTier()` (`web/src/lib/admission-blocks.js`), paired with a symbol so meaning is never colour-only. | Tier | Range | Vietnamese | @@ -150,7 +157,7 @@ symbol so meaning is never colour-only. ## Admission blocks Vietnamese universities admit on three-subject combinations (khối thi). -`src/lib/admission-blocks.js` lists the blocks computable from this schema +`web/src/lib/admission-blocks.js` lists the blocks computable from this schema (A00–A11, B00–B08, C00–C20, D01–D15, plus D05/D06 for German and Japanese). `computeBlocks(student)` returns those where all three scores exist, sorted by total descending. @@ -165,11 +172,11 @@ total descending. | Diacritics search | Pre-computed `ho_ten_ascii` | `LOWER(REPLACE(...))` at query time defeats the index | | SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes cannot persist; the allowlist prevents confusion | | Row caps | 100 (lookup), 1000 (SQL) | Keeps DOM render sizes reasonable | -| Routing | Hand-rolled, ~20 lines | Five static routes do not justify a router dependency | +| Routing | Hand-rolled, ~15 lines | Three static routes do not justify a router dependency | ## Risks and limitations -- **Database size.** 38–48 MB gzipped per dataset; slow links wait, mitigated by +- **Database size.** 44–48 MB gzipped per dataset; slow links wait, mitigated by a progress bar. - **Browser memory.** The full database lives in RAM; older mobile devices may run out. @@ -177,4 +184,4 @@ total descending. load. Self-hosting `sql-wasm.wasm` and updating `SQL_WASM_URL` in `use-sqlite.js` is the fix. - **Excel format drift.** A new source file with an unseen header layout needs a - new branch in `format_detect_2016.rs` or a new config. + new branch in `parser/internal/ingest/detect2016.go` or a new config. diff --git a/parser/Cargo.lock b/parser/Cargo.lock deleted file mode 100644 index e57f452..0000000 --- a/parser/Cargo.lock +++ /dev/null @@ -1,1159 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "calamine" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1" -dependencies = [ - "byteorder", - "codepage", - "encoding_rs", - "log", - "quick-xml", - "serde", - "zip", -] - -[[package]] -name = "cc" -version = "1.2.62" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - -[[package]] -name = "codepage" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" -dependencies = [ - "encoding_rs", -] - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "deflate64" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] - -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "find-msvc-tools" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" - -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" -dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.186" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" - -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", - "simd-adler32", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", - "hmac", -] - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "encoding_rs", - "memchr", -] - -[[package]] -name = "quote" -version = "1.0.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" - -[[package]] -name = "rusqlite" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "simd-adler32" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "time" -version = "0.3.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - -[[package]] -name = "typenum" -version = "1.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.121" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.121" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.121" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.121" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "xlsxread" -version = "0.1.0" -dependencies = [ - "anyhow", - "calamine", - "clap", - "glob", - "regex", - "rusqlite", - "serde", - "thiserror 1.0.69", - "toml", - "unicode-normalization", - "zip", -] - -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - -[[package]] -name = "zerocopy" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zip" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" -dependencies = [ - "aes", - "arbitrary", - "bzip2", - "constant_time_eq", - "crc32fast", - "crossbeam-utils", - "deflate64", - "displaydoc", - "flate2", - "getrandom", - "hmac", - "indexmap", - "lzma-rs", - "memchr", - "pbkdf2", - "sha1", - "thiserror 2.0.18", - "time", - "xz2", - "zeroize", - "zopfli", - "zstd", -] - -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/parser/Cargo.toml b/parser/Cargo.toml deleted file mode 100644 index d23dcca..0000000 --- a/parser/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "xlsxread" -version = "0.1.0" -edition = "2021" -description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016" - -[dependencies] -calamine = "0.26" -rusqlite = { version = "0.32", features = ["bundled"] } -clap = { version = "4", features = ["derive"] } -serde = { version = "1", features = ["derive"] } -toml = "0.8" -regex = "1" -unicode-normalization = "0.1" -thiserror = "1" -anyhow = "1" -glob = "0.3" - -# zip is already a transitive dep of calamine; pin explicitly so tests can use it -[dev-dependencies] -zip = "2" -rusqlite = { version = "0.32", features = ["bundled"] } - -[[bin]] -name = "xlsxread" -path = "src/main.rs" - -[[test]] -name = "golden" -path = "tests/golden.rs" diff --git a/parser/README.md b/parser/README.md new file mode 100644 index 0000000..519afd2 --- /dev/null +++ b/parser/README.md @@ -0,0 +1,76 @@ +# parser + +Reads the `.xls`/`.xlsx` source spreadsheets in `data/` and writes one SQLite +database per dataset. + +```bash +go -C parser build -o bin/xlsxread ./cmd/xlsxread # compile +go -C parser test ./... # unit tests + the reader-fidelity suite +``` + +``` +xlsxread build --schema parser/configs/.yml --input data/ --output +xlsxread audit --schema parser/configs/.yml --input data/ --db +``` + +This stage only produces a database. Verifying it against the expected row +count, compressing it and publishing it belong to `assembler/`, which compiles +this binary and drives it per dataset: + +```bash +go -C assembler run ./cmd/assemble db +``` + +## Layout + +| path | role | +|---|---| +| `internal/reader` | spreadsheet reading; the only place that knows about file formats | +| `internal/ingest` | dataset policy — sheet selection, header skipping, blank rows, the build loop, and the 2016 per-sheet format detection | +| `internal/transform` | `ToAscii`, score-regex parsing, row validation | +| `internal/schema` | the canonical 22-column table: DDL, INSERT, subject regexes | +| `internal/config` | per-dataset YAML parse rules | +| `internal/writer` | SQLite lifecycle and the stats block | +| `internal/audit` | source-vs-database SBD comparison | + +The reader deliberately knows nothing about datasets: it reports every sheet and +every row verbatim. All policy lives in `ingest`. That split is what made the +reader independently verifiable against a hash oracle. + +## Provenance + +This is a port of a Rust crate that occupied this same path until the Go +implementation reached full parity, when it was built alongside as `go-parser/` +and moved back here once the Rust was removed. Source comments cite the original +by file and line (`parser/src/transform.rs:56` and similar) — those refer to the +Rust tree and resolve at the tag **`pre-go-parser-removal`**, the last commit +containing it. + +The port was gated on a field-by-field comparison of both implementations across +the four datasets that existed then — 3,265,641 rows with identical full-table +SHA-256, identical per-column non-NULL counts, identical schema metadata and +identical build stdout. `scripts/differential-parity.mjs` is that comparator and +still runs against any two sets of databases. + +Behaviour was matched bug-for-bug, deliberately. Several quirks look like +defects and are load-bearing for the published data: + +- a parsed score of `0` becomes NULL in the 2016 separate-scores layout, + replicating a JavaScript falsy check; +- `ToAscii` strips combining marks in the literal range U+0300–U+036F rather + than by Unicode category, which is narrower; +- gender is a two-value allowlist, and anything else becomes NULL; +- `diem_thi` is read untrimmed while the other three fields are trimmed; +- the `"SINH "` header token carries a trailing space. + +Each has a test naming it, so none can be tidied away by accident. + +## Verification + +`testdata/reader-fidelity-hashes.tsv` holds a SHA-256 per input file over a +canonical dump of every cell of every sheet. It is **frozen**: it was produced +by the Rust reader, which no longer exists, so it cannot be regenerated. It +still fails if any single cell of any of the 182 files reads differently. + +The assembler refuses to publish a database whose row count does not match +the known figure, or whose artifact is under 90% of its usual size. diff --git a/parser/cmd/dumpcells/main.go b/parser/cmd/dumpcells/main.go new file mode 100644 index 0000000..e209b49 --- /dev/null +++ b/parser/cmd/dumpcells/main.go @@ -0,0 +1,88 @@ +// Command dumpcells emits the canonical cell rendering of a spreadsheet, for +// comparison against the Rust/calamine ground truth produced by +// parser/examples/dump_cells.rs. +// +// The canonical stream carries geometry and rendered cell values only. The +// calamine Data variant is deliberately excluded: Data::Empty and +// Data::String("") both render "" and both count as blank everywhere +// downstream, so the distinction cannot affect the database. +// +// Usage: dumpcells [out-file] +package main + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/tiennm99/thptqg/parser/internal/reader" +) + +// escape mirrors the Rust dumper so field separators can never break the format. +func escape(s string) string { + var b strings.Builder + b.Grow(len(s)) + for _, ch := range s { + switch ch { + case '\\': + b.WriteString(`\\`) + case '\t': + b.WriteString(`\t`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + default: + b.WriteRune(ch) + } + } + return b.String() +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: dumpcells [out-file]") + os.Exit(2) + } + path := os.Args[1] + + out := os.Stdout + if len(os.Args) > 2 { + f, err := os.Create(os.Args[2]) + if err != nil { + fmt.Fprintf(os.Stderr, "create: %v\n", err) + os.Exit(1) + } + defer f.Close() + out = f + } + w := bufio.NewWriterSize(out, 1<<20) + defer w.Flush() + + wb, err := reader.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, "open: %v\n", err) + os.Exit(1) + } + defer wb.Close() + + sheets := wb.Sheets() + fmt.Fprintf(w, "FILE\t%s\n", escape(path)) + fmt.Fprintf(w, "SHEETCOUNT\t%d\n", len(sheets)) + + for _, sh := range sheets { + fmt.Fprintf(w, "SHEET\t%d\t%s\t%d\t%d\n", sh.Index, escape(sh.Name), sh.Height, sh.Width) + err := wb.EachRow(sh.Index, func(s reader.Sheet, rowIdx int, row []reader.Cell) error { + fmt.Fprintf(w, "ROW\t%d\t%d\t%d\n", s.Index, rowIdx, len(row)) + for c, cell := range row { + fmt.Fprintf(w, "CELL\t%d\t%d\t%d\t%s\n", s.Index, rowIdx, c, escape(cell.Str)) + } + return nil + }) + if err != nil { + fmt.Fprintf(os.Stderr, "rows: %v\n", err) + os.Exit(1) + } + } +} diff --git a/parser/cmd/xlsxread/main.go b/parser/cmd/xlsxread/main.go new file mode 100644 index 0000000..cf7dc89 --- /dev/null +++ b/parser/cmd/xlsxread/main.go @@ -0,0 +1,115 @@ +// Command xlsxread reads .xls/.xlsx files and builds SQLite databases for the +// thptqg datasets. +// +// The CLI contract is fixed by parser/scripts/build-db.js and must match the +// Rust binary exactly: +// +// xlsxread build --schema --input

--output +// xlsxread audit --schema --input --db +// +// Implemented with the standard flag package rather than a CLI framework: two +// subcommands with three flags each do not justify a dependency. +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/tiennm99/thptqg/parser/internal/audit" + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/ingest" +) + +func usage() { + fmt.Fprint(os.Stderr, `xlsxread — read .xls/.xlsx files and build SQLite databases for thptqg datasets + +Usage: + xlsxread build --schema --input --output + xlsxread audit --schema --input --db +`) +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + + switch os.Args[1] { + case "build": + runBuild(os.Args[2:]) + case "audit": + runAudit(os.Args[2:]) + case "-h", "--help", "help": + usage() + default: + fmt.Fprintf(os.Stderr, "unknown subcommand %q\n\n", os.Args[1]) + usage() + os.Exit(2) + } +} + +func runBuild(args []string) { + fs := flag.NewFlagSet("build", flag.ExitOnError) + schemaPath := fs.String("schema", "", "path to the dataset YAML config file") + inputDir := fs.String("input", "", "directory containing the .xls / .xlsx source files") + outputPath := fs.String("output", "", "output SQLite database path") + fs.Parse(args) + + if *schemaPath == "" || *inputDir == "" || *outputPath == "" { + fmt.Fprintln(os.Stderr, "build requires --schema, --input and --output") + os.Exit(2) + } + + cfg, err := config.Load(*schemaPath) + if err != nil { + fatalf("Failed to load config: %v", err) + } + + // The 2016 dataset selects its column layout per file at runtime; every other + // dataset uses the fixed columns: mapping (main.rs:63-67). + if cfg.FormatDetection != nil && *cfg.FormatDetection == "thptqg2016" { + if err := ingest.Detect2016(cfg, *inputDir, *outputPath); err != nil { + fatalf("%v", err) + } + return + } + if err := ingest.Standard(cfg, *inputDir, *outputPath); err != nil { + fatalf("%v", err) + } +} + +func runAudit(args []string) { + fs := flag.NewFlagSet("audit", flag.ExitOnError) + schemaPath := fs.String("schema", "", "path to the dataset YAML config file") + inputDir := fs.String("input", "", "directory containing the .xlsx source files") + dbPath := fs.String("db", "", "SQLite database to compare against") + fs.Parse(args) + + if *schemaPath == "" || *inputDir == "" || *dbPath == "" { + fmt.Fprintln(os.Stderr, "audit requires --schema, --input and --db") + os.Exit(2) + } + + cfg, err := config.Load(*schemaPath) + if err != nil { + fatalf("Failed to load config: %v", err) + } + res, err := audit.Run(*inputDir, *dbPath, cfg) + if err != nil { + fatalf("%v", err) + } + audit.PrintReport(res) + + // A mismatch is a non-zero exit even though the audit itself succeeded + // (main.rs:46-48) — CI treats it as a failure signal. + if !res.Matched { + os.Exit(1) + } +} + +func fatalf(format string, a ...any) { + fmt.Fprintf(os.Stderr, "Error: "+format+"\n", a...) + os.Exit(1) +} diff --git a/parser/configs/2016.toml b/parser/configs/2016.yml similarity index 60% rename from parser/configs/2016.toml rename to parser/configs/2016.yml index 6bde781..b2a938d 100644 --- a/parser/configs/2016.toml +++ b/parser/configs/2016.yml @@ -1,7 +1,7 @@ # thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files. # # Three column layouts exist across the 119 files; the binary selects the -# right one per-file at runtime via format_detection = "thptqg2016": +# right one per-file at runtime via format_detection: thptqg2016: # # separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files # mapped header SOBAODANH|SBD + DIEM_THI — most provinces @@ -13,20 +13,20 @@ # sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap). # strip_blank_rows = false: no blank-row anomaly observed in this dataset. # -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. +# Table shape, INSERT and subject regexes are canonical — see parser/internal/schema/schema.go. -format_detection = "thptqg2016" +format_detection: thptqg2016 -[reader] -sheet_mode = "all" -strip_blank_rows = false +reader: + sheet_mode: all + strip_blank_rows: false -[validation] -require_numeric_sbd = false -require_nonempty_name = true -require_nonempty_sbd = true +validation: + require_numeric_sbd: false + require_nonempty_name: true + require_nonempty_sbd: true -[header] -# Tokens that identify a header row by first-cell content (uppercased). -# Covers both SOBAODANH-style and SBD-style headers. -tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"] +header: + # Tokens that identify a header row by first-cell content (uppercased). + # Covers both SOBAODANH-style and SBD-style headers. + tokens: ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"] diff --git a/parser/configs/2017-old.toml b/parser/configs/2017-old.toml deleted file mode 100644 index f21f8aa..0000000 --- a/parser/configs/2017-old.toml +++ /dev/null @@ -1,25 +0,0 @@ -# thptqg2017 data-old/ — 63 .xlsx files (pre-baotintuc refresh). -# -# sheet_mode = "first": single-sheet workbooks, never hit the 65k row cap. -# SBD validation: require ^\d+$ (build-database-old.js:55 guard). -# strip_blank_rows = false: no explicit blank-skip in build-database-old.js. -# -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. - -[reader] -sheet_mode = "first" -strip_blank_rows = false - -[columns] -ho_ten = 0 -ngay_sinh = 1 -so_bao_danh = 2 -diem_thi = 3 - -[validation] -require_numeric_sbd = true -require_nonempty_name = true -require_nonempty_sbd = true - -[header] -tokens = ["HO_TEN", "HỌ TÊN", "STT"] diff --git a/parser/configs/2017-old2.toml b/parser/configs/2017-old2.toml deleted file mode 100644 index c4fd6b7..0000000 --- a/parser/configs/2017-old2.toml +++ /dev/null @@ -1,26 +0,0 @@ -# thptqg2017 data-old2/ — 54 .xlsx files (corrected-export set). -# -# sheet_mode = "all": 24.HCM_UTLQ.xlsx overflows into Sheet2 (+6,446 rows). -# SBD validation: require ^\d+$ (build-database-old2.js:57 guard). -# strip_blank_rows = true: skip fully blank rows BEFORE counting sourceRows -# (build-database-old2.js:50-51 checks blank before sourceRows++). -# -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. - -[reader] -sheet_mode = "all" -strip_blank_rows = true - -[columns] -ho_ten = 0 -ngay_sinh = 1 -so_bao_danh = 2 -diem_thi = 3 - -[validation] -require_numeric_sbd = true -require_nonempty_name = true -require_nonempty_sbd = true - -[header] -tokens = ["HO_TEN", "HỌ TÊN", "STT"] diff --git a/parser/configs/2017.toml b/parser/configs/2017.yml similarity index 55% rename from parser/configs/2017.toml rename to parser/configs/2017.yml index 99dccbb..ea87f94 100644 --- a/parser/configs/2017.toml +++ b/parser/configs/2017.yml @@ -4,22 +4,22 @@ # SBD validation: no numeric guard — build-database.js did not apply ^\d+$. # strip_blank_rows = false: no blank-row anomaly in this dataset. # -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. +# Table shape, INSERT and subject regexes are canonical — see parser/internal/schema/schema.go. -[reader] -sheet_mode = "all" -strip_blank_rows = false +reader: + sheet_mode: all + strip_blank_rows: false -[columns] -ho_ten = 0 -ngay_sinh = 1 -so_bao_danh = 2 -diem_thi = 3 +columns: + ho_ten: 0 + ngay_sinh: 1 + so_bao_danh: 2 + diem_thi: 3 -[validation] -require_numeric_sbd = false -require_nonempty_name = true -require_nonempty_sbd = true +validation: + require_numeric_sbd: false + require_nonempty_name: true + require_nonempty_sbd: true -[header] -tokens = ["HO_TEN", "HỌ TÊN", "STT"] +header: + tokens: ["HO_TEN", "HỌ TÊN", "STT"] diff --git a/parser/go.mod b/parser/go.mod new file mode 100644 index 0000000..7ad9076 --- /dev/null +++ b/parser/go.mod @@ -0,0 +1,30 @@ +module github.com/tiennm99/thptqg/parser + +go 1.26.5 + +require ( + github.com/pbnjay/grate v0.0.0-20231006022435-3f8e65d74a14 + github.com/xuri/excelize/v2 v2.11.0 + golang.org/x/text v0.41.0 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.56.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.7 // indirect + github.com/richardlehane/msoleps v1.0.6 // indirect + github.com/tiendc/go-deepcopy v1.7.2 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/parser/go.sum b/parser/go.sum new file mode 100644 index 0000000..ef51566 --- /dev/null +++ b/parser/go.sum @@ -0,0 +1,82 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pbnjay/grate v0.0.0-20231006022435-3f8e65d74a14 h1:ZfXdW7GIVZT3Z9oejLJ+GHrrQv/ezU2Bwqn0BF37s4g= +github.com/pbnjay/grate v0.0.0-20231006022435-3f8e65d74a14/go.mod h1:VaZEKQrYbYr2untVA/EFNdC6hM7GyARRNM+k4+5CmA0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.7 h1:oeoiM0WE79vHwE8RpIYYvIAc8ajTH2mb6UZm55/+EB0= +github.com/richardlehane/mscfb v1.0.7/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= +github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= +github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44= +github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG88= +github.com/xuri/excelize/v2 v2.11.0/go.mod h1:jxFLbzaIwGQ5ufFNvYfUOHqXhfPaNmP14KWfmNz2Uak= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= +golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/parser/internal/audit/audit.go b/parser/internal/audit/audit.go new file mode 100644 index 0000000..7185ef5 --- /dev/null +++ b/parser/internal/audit/audit.go @@ -0,0 +1,148 @@ +// Package audit compares distinct SBDs in the source spreadsheets against the +// row count in a built database — a port of parser/src/audit.rs. +// +// Two deliberate divergences from the build path are preserved, both inherited +// from audit-row-counts.js: +// +// - only .xlsx files are considered (audit.rs:53-59) +// - only sheet 0 is read, regardless of sheet_mode (audit.rs:81-84) +// +// These are intentional, not oversights. Do not "fix" them. +package audit + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/ingest" + "github.com/tiennm99/thptqg/parser/internal/reader" + "github.com/tiennm99/thptqg/parser/internal/sqlitedb" +) + +// Result carries the audit counters. +type Result struct { + TotalDataRows uint64 + BothEmpty uint64 + EmptyName uint64 + EmptySbd uint64 + DistinctSbds int + DBCount int64 + Matched bool +} + +// Run collects distinct SBDs from the .xlsx files in inputDir and compares the +// total against the student row count in dbPath. +func Run(inputDir, dbPath string, cfg *config.DatasetConfig) (*Result, error) { + entries, err := os.ReadDir(inputDir) + if err != nil { + return nil, fmt.Errorf("read input dir %s: %w", inputDir, err) + } + var files []string + for _, e := range entries { + if e.IsDir() { + continue + } + if strings.EqualFold(filepath.Ext(e.Name()), ".xlsx") { + files = append(files, filepath.Join(inputDir, e.Name())) + } + } + sort.Strings(files) + + res := &Result{} + seen := make(map[string]struct{}) + + // The audit uses positional defaults for format-detection configs, mirroring + // audit-row-counts.js's fixed column assumption (audit.rs:104-111). + hoTenCol, sbdCol := 1, 0 + if cfg.Columns != nil { + hoTenCol, sbdCol = cfg.Columns.HoTen, cfg.Columns.SoBaoDanh + } + + for _, file := range files { + wb, err := reader.Open(file) + if err != nil { + return nil, err + } + sheets := wb.Sheets() + if len(sheets) == 0 { + wb.Close() + continue + } + + firstRow := true + err = wb.EachRow(sheets[0].Index, func(_ reader.Sheet, _ int, row []reader.Cell) error { + if firstRow { + firstRow = false + if ingest.IsHeaderRow(row, cfg.Header.Tokens) { + return nil + } + } + res.TotalDataRows++ + + hoTen := cellAt(row, hoTenCol) + sbd := cellAt(row, sbdCol) + + if hoTen == "" && sbd == "" { + res.BothEmpty++ + return nil + } + if hoTen == "" { + res.EmptyName++ + } + if sbd == "" { + res.EmptySbd++ + } + if sbd != "" { + seen[sbd] = struct{}{} + } + return nil + }) + wb.Close() + if err != nil { + return nil, err + } + } + + // Opened read-only: the audit must never mutate the database it inspects + // (audit.rs:139). + db, err := sql.Open(sqlitedb.DriverName, "file:"+dbPath+"?mode=ro") + if err != nil { + return nil, fmt.Errorf("open db %s: %w", dbPath, err) + } + defer db.Close() + if err := db.QueryRow("SELECT COUNT(*) FROM student").Scan(&res.DBCount); err != nil { + return nil, fmt.Errorf("count rows: %w", err) + } + + res.DistinctSbds = len(seen) + res.Matched = int64(res.DistinctSbds) == res.DBCount + return res, nil +} + +// PrintReport mirrors audit-row-counts.js:54-62 exactly. +func PrintReport(r *Result) { + fmt.Println("=== Source vs DB ===") + fmt.Printf("Source: total data rows across all files: %d\n", r.TotalDataRows) + fmt.Printf("Source: rows with empty name AND sbd (skipped): %d\n", r.BothEmpty) + fmt.Printf("Source: rows with missing name only: %d\n", r.EmptyName) + fmt.Printf("Source: rows with missing sbd only: %d\n", r.EmptySbd) + fmt.Printf("Source: distinct SBDs: %d\n", r.DistinctSbds) + fmt.Printf("DB: row count: %d\n", r.DBCount) + if r.Matched { + fmt.Println("Match: YES — all unique SBDs accounted for") + } else { + fmt.Printf("Match: NO — gap of %d\n", int64(r.DistinctSbds)-r.DBCount) + } +} + +func cellAt(row []reader.Cell, idx int) string { + if idx < 0 || idx >= len(row) { + return "" + } + return strings.TrimSpace(row[idx].Str) +} diff --git a/parser/internal/config/config.go b/parser/internal/config/config.go new file mode 100644 index 0000000..2860990 --- /dev/null +++ b/parser/internal/config/config.go @@ -0,0 +1,113 @@ +// Package config loads per-dataset parse rules — a port of parser/src/config.rs. +// +// The config deliberately carries no SQL. The table shape, the INSERT and the +// subject regexes are identical for every dataset and live in internal/schema; +// keeping them here meant four copies of the same DDL, which is how the 2016 and +// 2017 schemas drifted apart. +package config + +import ( + "bytes" + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +// SheetMode selects which sheets of a workbook are read. +type SheetMode string + +const ( + // SheetModeAll iterates every sheet, which is what recovers the Hanoi and + // HCM rows that overflow past Excel's 65,536-row cap into a second sheet. + SheetModeAll SheetMode = "all" + // SheetModeFirst reads sheet 0 only. + SheetModeFirst SheetMode = "first" +) + +// valid reports whether m is one of the two values the Rust enum accepts. +// +// Checked after decoding rather than via a custom unmarshaler: the decoder +// assigns named string types directly, so a typo would otherwise decode +// silently and be read as "not all" downstream. +func (m SheetMode) valid() bool { + return m == SheetModeAll || m == SheetModeFirst +} + +// DatasetConfig is the per-dataset parse rule set. +type DatasetConfig struct { + Reader ReaderCfg `yaml:"reader"` + // Columns holds fixed column indices. Nil when FormatDetection handles + // per-file mapping — a pointer rather than a value because a zero ColumnMap + // would silently mean "every column is index 0". + Columns *ColumnMap `yaml:"columns"` + Validation ValidationCfg `yaml:"validation"` + Header HeaderCfg `yaml:"header"` + // FormatDetection, when set to "thptqg2016", enables per-file format + // auto-detection: each file's header row is inspected at runtime to choose + // the column layout (separate-scores / mapped / default-positional). + FormatDetection *string `yaml:"format_detection"` +} + +type ReaderCfg struct { + SheetMode SheetMode `yaml:"sheet_mode"` + // StripBlankRows skips rows where every cell is empty before counting them + // as source rows (a 2017-old2 quirk). + StripBlankRows bool `yaml:"strip_blank_rows"` +} + +// ColumnMap holds zero-indexed column positions in the source row. Used by the +// 2017 configs; 2016 uses runtime format detection instead. +type ColumnMap struct { + HoTen int `yaml:"ho_ten"` + NgaySinh int `yaml:"ngay_sinh"` + SoBaoDanh int `yaml:"so_bao_danh"` + DiemThi int `yaml:"diem_thi"` +} + +type ValidationCfg struct { + // RequireNumericSbd mirrors build-database-old.js / -old2.js, which require + // so_bao_danh to match ^\d+$. + RequireNumericSbd bool `yaml:"require_numeric_sbd"` + RequireNonemptyName bool `yaml:"require_nonempty_name"` + RequireNonemptySbd bool `yaml:"require_nonempty_sbd"` +} + +type HeaderCfg struct { + // Tokens are matched against the uppercased first cell to detect a header row. + Tokens []string `yaml:"tokens"` +} + +// Parse decodes a config, rejecting any key the struct does not declare. +// +// Strictness is load-bearing and has its own test. Rust gets it from serde's +// deny_unknown_fields; yaml.v3 needs KnownFields(true) explicitly, and Go YAML +// decoders ignore unknown keys by default. Without it a leftover `schema:` +// mapping would look effective while internal/schema actually drove the build — +// the drift that produced two divergent schemas before the unification. +func Parse(src []byte) (*DatasetConfig, error) { + var cfg DatasetConfig + dec := yaml.NewDecoder(bytes.NewReader(src)) + dec.KnownFields(true) + if err := dec.Decode(&cfg); err != nil { + return nil, fmt.Errorf("decode config: %w", err) + } + if !cfg.Reader.SheetMode.valid() { + return nil, fmt.Errorf("invalid sheet_mode %q (want %q or %q)", + cfg.Reader.SheetMode, SheetModeAll, SheetModeFirst) + } + return &cfg, nil +} + +// Load reads and parses a config file. +func Load(path string) (*DatasetConfig, error) { + src, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read config %s: %w", path, err) + } + cfg, err := Parse(src) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return cfg, nil +} diff --git a/parser/internal/config/config_test.go b/parser/internal/config/config_test.go new file mode 100644 index 0000000..9e7e195 --- /dev/null +++ b/parser/internal/config/config_test.go @@ -0,0 +1,224 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// sampleYAML mirrors SAMPLE_YAML in parser/src/config.rs. +const sampleYAML = ` +reader: + sheet_mode: all + strip_blank_rows: false + +columns: + ho_ten: 0 + ngay_sinh: 1 + so_bao_danh: 2 + diem_thi: 3 + +validation: + require_numeric_sbd: false + require_nonempty_name: true + require_nonempty_sbd: true + +header: + tokens: ["HO_TEN", "HỌ TÊN", "STT"] +` + +// TestConfigRoundTrip ports config_round_trip (config.rs:115). +func TestConfigRoundTrip(t *testing.T) { + cfg, err := Parse([]byte(sampleYAML)) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + if cfg.Reader.SheetMode != SheetModeAll { + t.Errorf("sheet_mode = %q, want all", cfg.Reader.SheetMode) + } + if cfg.Reader.StripBlankRows { + t.Error("strip_blank_rows should be false") + } + if cfg.Columns == nil { + t.Fatal("columns should be present") + } + if cfg.Columns.HoTen != 0 { + t.Errorf("ho_ten = %d, want 0", cfg.Columns.HoTen) + } + if cfg.Columns.DiemThi != 3 { + t.Errorf("diem_thi = %d, want 3", cfg.Columns.DiemThi) + } + if cfg.Validation.RequireNumericSbd { + t.Error("require_numeric_sbd should be false") + } + if !cfg.Validation.RequireNonemptyName { + t.Error("require_nonempty_name should be true") + } + if len(cfg.Header.Tokens) != 3 { + t.Errorf("tokens = %d, want 3", len(cfg.Header.Tokens)) + } + if cfg.FormatDetection != nil { + t.Errorf("format_detection = %v, want nil", *cfg.FormatDetection) + } +} + +// TestConfigRejectsLeftoverSQLSections ports config_rejects_leftover_sql_sections +// (config.rs:132). This is the load-bearing one: Rust gets the behaviour free +// from serde's deny_unknown_fields, whereas Go YAML decoders ignore unknown keys +// unless KnownFields(true) is set. Without it a stale schema: mapping would look effective while +// internal/schema silently drove the build — exactly the drift that produced two +// divergent schemas before the unification. +func TestConfigRejectsLeftoverSQLSections(t *testing.T) { + withDDL := sampleYAML + "\nschema:\n ddl: \"CREATE TABLE student (so_bao_danh TEXT);\"\n" + if _, err := Parse([]byte(withDDL)); err == nil { + t.Fatal("config with a leftover schema: mapping must be rejected") + } +} + +// TestConfigRejectsUnknownScalarKey guards the same property for a stray scalar, +// not just a stray mapping. +func TestConfigRejectsUnknownScalarKey(t *testing.T) { + withKey := sampleYAML + "\nunexpected_key: 1\n" + if _, err := Parse([]byte(withKey)); err == nil { + t.Fatal("config with an unknown scalar key must be rejected") + } +} + +// TestConfigFirstSheetMode ports config_first_sheet_mode (config.rs:140). +func TestConfigFirstSheetMode(t *testing.T) { + src := []byte(replaceAll(sampleYAML, "sheet_mode: all", "sheet_mode: first")) + cfg, err := Parse(src) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + if cfg.Reader.SheetMode != SheetModeFirst { + t.Errorf("sheet_mode = %q, want first", cfg.Reader.SheetMode) + } +} + +// TestConfigRejectsUnknownSheetMode: the Rust enum accepts only "all"/"first", +// so anything else must fail rather than defaulting. +func TestConfigRejectsUnknownSheetMode(t *testing.T) { + src := []byte(replaceAll(sampleYAML, "sheet_mode: all", "sheet_mode: second")) + if _, err := Parse(src); err == nil { + t.Fatal("unknown sheet_mode must be rejected") + } +} + +// TestConfigFormatDetectionField ports config_format_detection_field +// (config.rs:147): a 2016-style config has no columns: mapping at all. +func TestConfigFormatDetectionField(t *testing.T) { + const src = ` +format_detection: thptqg2016 + +reader: + sheet_mode: all + strip_blank_rows: false + +validation: + require_numeric_sbd: false + require_nonempty_name: true + require_nonempty_sbd: true + +header: + tokens: ["SBD", "SOBAODANH", "STT"] +` + cfg, err := Parse([]byte(src)) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + if cfg.FormatDetection == nil || *cfg.FormatDetection != "thptqg2016" { + t.Errorf("format_detection = %v, want thptqg2016", cfg.FormatDetection) + } + if cfg.Columns != nil { + t.Error("columns must be nil when format_detection drives the layout") + } +} + +// TestLoadRealConfigs loads the shipped configs and asserts the per-dataset +// differences recorded during scouting. +func TestLoadRealConfigs(t *testing.T) { + root := repoRoot(t) + want := map[string]struct { + sheetMode SheetMode + stripBlank bool + numericSbd bool + hasColumns bool + formatDet string + tokenCount int + }{ + "2016": {SheetModeAll, false, false, false, "thptqg2016", 6}, + "2017": {SheetModeAll, false, false, true, "", 3}, + } + for id, w := range want { + t.Run(id, func(t *testing.T) { + cfg, err := Load(filepath.Join(root, "parser", "configs", id+".yml")) + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.Reader.SheetMode != w.sheetMode { + t.Errorf("sheet_mode = %q, want %q", cfg.Reader.SheetMode, w.sheetMode) + } + if cfg.Reader.StripBlankRows != w.stripBlank { + t.Errorf("strip_blank_rows = %v, want %v", cfg.Reader.StripBlankRows, w.stripBlank) + } + if cfg.Validation.RequireNumericSbd != w.numericSbd { + t.Errorf("require_numeric_sbd = %v, want %v", cfg.Validation.RequireNumericSbd, w.numericSbd) + } + if (cfg.Columns != nil) != w.hasColumns { + t.Errorf("columns present = %v, want %v", cfg.Columns != nil, w.hasColumns) + } + got := "" + if cfg.FormatDetection != nil { + got = *cfg.FormatDetection + } + if got != w.formatDet { + t.Errorf("format_detection = %q, want %q", got, w.formatDet) + } + if len(cfg.Header.Tokens) != w.tokenCount { + t.Errorf("tokens = %d, want %d", len(cfg.Header.Tokens), w.tokenCount) + } + // Every dataset requires a non-empty name and SBD. + if !cfg.Validation.RequireNonemptyName || !cfg.Validation.RequireNonemptySbd { + t.Error("both non-empty validations should be true for every dataset") + } + }) + } +} + +func replaceAll(s, old, new string) string { + out := "" + for { + i := indexOf(s, old) + if i < 0 { + return out + s + } + out += s[:i] + new + s = s[i+len(old):] + } +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} + +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(dir, "parser", "configs")); err == nil { + return dir + } + dir = filepath.Dir(dir) + } + t.Fatal("could not locate repo root") + return "" +} diff --git a/parser/internal/ingest/detect2016.go b/parser/internal/ingest/detect2016.go new file mode 100644 index 0000000..1cf43a2 --- /dev/null +++ b/parser/internal/ingest/detect2016.go @@ -0,0 +1,382 @@ +package ingest + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/reader" + "github.com/tiennm99/thptqg/parser/internal/transform" + "github.com/tiennm99/thptqg/parser/internal/writer" +) + +// The 2016 dataset's 119 files were produced by inconsistent tooling and use +// three different column layouts, chosen per sheet at runtime. This is a port of +// parser/src/format_detect_2016.rs — institutional knowledge encoded as +// literals, with no abstraction to derive it from, so everything here is copied +// verbatim rather than rationalised. + +// KnownHeaders are the upper-cased first-cell values that identify a header row +// (format_detect_2016.rs:36-54, mirroring KNOWN_HEADERS in build-database.js). +// +// NOTE "SINH " carries a TRAILING SPACE, exactly as in the Rust source. Trimming +// it would change which rows are recognised as headers. +var KnownHeaders = []string{ + "SOBAODANH", + "SBD", + "HO_TEN", + "HOTEN", + "HỌ TÊN", + "NGAY_SINH", + "TEN_CUMTHI", + "GIOI_TINH", + "DIEM_THI", + "STT", + "TOAN", + "VAN", + "LY", + "HOA", + "SINH ", + "SU", + "DIA", +} + +func isKnownHeader(s string) bool { + for _, h := range KnownHeaders { + if h == s { + return true + } + } + return false +} + +// IsHeaderRow2016 reports whether row[0] is a known header token +// (format_detect_2016.rs:58-64). +// +// The guard here is len < 2, not the len < 3 used by the 2017 header check. +func IsHeaderRow2016(row []reader.Cell) bool { + if len(row) < 2 { + return false + } + return isKnownHeader(strings.ToUpper(strings.TrimSpace(row[0].Str))) +} + +// FormatKind is one of the three 2016 layouts. +type FormatKind int + +const ( + // FormatSeparateScores has one column per subject rather than a free-text + // DIEM_THI cell — the dhhanghai-style files. + FormatSeparateScores FormatKind = iota + // FormatMapped resolves column indices from the header by name. + FormatMapped + // FormatDefault is the positional 6-column layout used when no header is + // recognised. It is FormatMapped with fixed indices, not a separate path. + FormatDefault +) + +// Format is a detected layout plus, for the mapped case, its column indices. +type Format struct { + Kind FormatKind + Sbd int + HoTen int + NgaySinh *int + TenCumThi *int + GioiTinh *int + DiemThi int +} + +// defaultFormat is FormatMapped with the fixed positional indices +// (format_detect_2016.rs:295-306). +func defaultFormat() Format { + two, three, four := 2, 3, 4 + return Format{ + Kind: FormatDefault, Sbd: 0, HoTen: 1, + NgaySinh: &two, TenCumThi: &three, GioiTinh: &four, DiemThi: 5, + } +} + +// DetectFormat inspects a header row and decides which layout applies +// (format_detect_2016.rs:95-145). +func DetectFormat(headerRow []reader.Cell) Format { + cols := make([]string, len(headerRow)) + for i, c := range headerRow { + cols[i] = strings.ToUpper(strings.TrimSpace(c.Str)) + } + + // Format 1: SBD in col 0 AND TOAN in col 2. + if len(cols) > 2 && cols[0] == "SBD" && cols[2] == "TOAN" { + return Format{Kind: FormatSeparateScores} + } + + // Format 2: resolve indices by header name, order-independent. + var sbdIdx, hoTenIdx, ngaySinhIdx, tenCumThiIdx, gioiTinhIdx, diemThiIdx *int + for i := range cols { + idx := i + switch cols[i] { + case "SOBAODANH", "SBD": + sbdIdx = &idx + case "HO_TEN", "HOTEN", "HỌ TÊN": + hoTenIdx = &idx + case "NGAY_SINH": + ngaySinhIdx = &idx + case "TEN_CUMTHI": + tenCumThiIdx = &idx + case "GIOI_TINH": + gioiTinhIdx = &idx + case "DIEM_THI": + diemThiIdx = &idx + } + } + + if sbdIdx != nil && diemThiIdx != nil { + hoTen := 1 // fallback: col 1, present in all known files + if hoTenIdx != nil { + hoTen = *hoTenIdx + } + return Format{ + Kind: FormatMapped, Sbd: *sbdIdx, HoTen: hoTen, + NgaySinh: ngaySinhIdx, TenCumThi: tenCumThiIdx, GioiTinh: gioiTinhIdx, + DiemThi: *diemThiIdx, + } + } + + // Unrecognised header, or none at all. + return defaultFormat() +} + +// parseFloatCell parses a per-subject score cell. +// +// A parsed 0.0 becomes "no score" — this replicates JavaScript's +// `parseFloat(row[N]) || null`, where 0 is falsy (format_detect_2016.rs:165). +// It means a genuine zero is indistinguishable from a blank. Not obviously +// correct, but it is the shipped behaviour and the published data depends on it. +func parseFloatCell(row []reader.Cell, idx int) (float64, bool) { + s := cellAt(row, idx) + if s == "" { + return 0, false + } + v, err := strconv.ParseFloat(s, 64) + if err != nil || v == 0.0 { + return 0, false + } + return v, true +} + +// processSeparateScoresRow handles the fixed 12-column layout +// (format_detect_2016.rs:176-216): +// +// 0=SBD 1=HOTEN 2=TOAN 3=VAN 4=LY 5=HOA 6=SINH 7=SU 8=DIA +// 9=NGOAINGUTN 10=NGOAINGUTL 11=NGOAINGU(total -> tieng_anh) +// +// tieng_phap / tieng_duc / tieng_nhat / tieng_trung are structurally unreachable +// in this format, and ngay_sinh / ten_cum_thi / gioi_tinh are always nil. +// Scores are read as floats directly; this layout has no free-text score cell, +// so the subject regexes never run. +func processSeparateScoresRow(row []reader.Cell) *transform.ParsedRow { + sbd := cellAt(row, 0) + hoTen := cellAt(row, 1) + if sbd == "" || hoTen == "" { + return nil + } + + scores := make(map[string]float64) + for field, idx := range map[string]int{ + "toan": 2, "ngu_van": 3, "vat_ly": 4, "hoa_hoc": 5, + "sinh_hoc": 6, "lich_su": 7, "dia_ly": 8, + "tieng_anh": 11, // NGOAINGU total + } { + if v, ok := parseFloatCell(row, idx); ok { + scores[field] = v + } + } + + return &transform.ParsedRow{ + SoBaoDanh: sbd, + HoTen: hoTen, + HoTenAscii: transform.ToAscii(hoTen), + Scores: scores, + } +} + +// processMappedRow handles header-derived column indices +// (format_detect_2016.rs:226-289). +func processMappedRow(row []reader.Cell, f Format) *transform.ParsedRow { + sbd := cellAt(row, f.Sbd) + hoTen := cellAt(row, f.HoTen) + if sbd == "" || hoTen == "" { + return nil + } + + // Leaked-header guard: a row whose SBD or name cell is itself a header token + // is a repeated header, not data (format_detect_2016.rs:244-250). + if isKnownHeader(strings.ToUpper(sbd)) || isKnownHeader(strings.ToUpper(hoTen)) { + return nil + } + + optional := func(idx *int) *string { + if idx == nil { + return nil + } + if s := cellAt(row, *idx); s != "" { + return &s + } + return nil + } + + // Gender is a two-value allowlist, not a general enum: anything other than + // exactly "Nam" or "Nữ" becomes nil (format_detect_2016.rs:263-271). + var gioiTinh *string + if f.GioiTinh != nil { + if s := cellAt(row, *f.GioiTinh); s == "Nam" || s == "Nữ" { + gioiTinh = &s + } + } + + // Read untrimmed, matching format_detect_2016.rs:273-276. + diemThi := "" + if f.DiemThi >= 0 && f.DiemThi < len(row) { + diemThi = row[f.DiemThi].Str + } + + return &transform.ParsedRow{ + SoBaoDanh: sbd, + HoTen: hoTen, + HoTenAscii: transform.ToAscii(hoTen), + NgaySinh: optional(f.NgaySinh), + TenCumThi: optional(f.TenCumThi), + GioiTinh: gioiTinh, + Scores: transform.ParseScores(diemThi), + } +} + +// ProcessRow2016 dispatches a data row through the detected layout. A nil return +// means the row is empty or invalid and should be skipped. +func ProcessRow2016(row []reader.Cell, f Format) *transform.ParsedRow { + if f.Kind == FormatSeparateScores { + return processSeparateScoresRow(row) + } + // FormatMapped and FormatDefault share one implementation; Default is just a + // fixed index tuple. + return processMappedRow(row, f) +} + +// Detect2016 ingests the 2016 dataset — the port of run_build_2016 and +// process_file_2016 (main.rs:211-377). +func Detect2016(cfg *config.DatasetConfig, inputDir, outputPath string) error { + files, err := InputFiles(inputDir) + if err != nil { + return err + } + label := DatasetLabel(inputDir) + fmt.Printf("[build:2016] %s/ → %s (%d files)\n", label, outputPath, len(files)) + + db, err := writer.OpenDB(outputPath) + if err != nil { + return err + } + defer db.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin: %w", err) + } + ins, err := writer.Prepare(tx) + if err != nil { + tx.Rollback() + return err + } + + var st writer.Stats // Skipped stays 0 on this path, matching main.rs:251 + for _, file := range files { + base := filepath.Base(file) + fileRows, err := processFile2016(file, cfg, ins, base, &st) + if err != nil { + fmt.Fprintf(os.Stderr, " [error] %s: %v\n", base, err) + st.Errors++ + continue + } + fmt.Printf(" %s: %d rows\n", base, fileRows) + } + + if err := ins.Close(); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + return writer.Finish(db, outputPath, st, label, false) +} + +// processFile2016 detects the layout per SHEET and processes that sheet's rows. +// +// Detection is per sheet, not per file (main.rs:344-349): within one workbook, +// sheet 2 may legitimately detect differently from sheet 1, which matters for +// the province files that overflow past Excel's row cap. +func processFile2016(path string, cfg *config.DatasetConfig, ins *writer.Inserter, base string, st *writer.Stats) (uint64, error) { + wb, err := reader.Open(path) + if err != nil { + return 0, err + } + defer wb.Close() + + sheets := wb.Sheets() + if len(sheets) == 0 { + return 0, nil + } + if cfg.Reader.SheetMode == config.SheetModeFirst { + sheets = sheets[:1] + } + + var fileRows uint64 + for _, sh := range sheets { + var rows [][]reader.Cell + if err := wb.EachRow(sh.Index, func(_ reader.Sheet, _ int, row []reader.Cell) error { + rows = append(rows, row) + return nil + }); err != nil { + return fileRows, err + } + if len(rows) == 0 { + continue + } + + f := defaultFormat() + startIdx := 0 + if IsHeaderRow2016(rows[0]) { + f = DetectFormat(rows[0]) + startIdx = 1 + } + + for _, row := range rows[startIdx:] { + // Rows shorter than 2 cells are dropped BEFORE the counter + // (main.rs:351-353), so they never appear in the source total. + if len(row) < 2 { + continue + } + st.SourceRows++ + + parsed := ProcessRow2016(row, f) + if parsed == nil { + // Empty or invalid. Note this is NOT counted as skipped — the + // 2016 path leaves that counter at zero (main.rs:251), so the + // stats block reports insertable == source rows and the Audit + // line absorbs the difference. + continue + } + if err := ins.Insert(parsed); err != nil { + st.Errors++ + if st.Errors <= 5 { + fmt.Fprintf(os.Stderr, " [warn] %s: %v\n", base, err) + } + continue + } + fileRows++ + } + } + return fileRows, nil +} diff --git a/parser/internal/ingest/detect2016_test.go b/parser/internal/ingest/detect2016_test.go new file mode 100644 index 0000000..93c3b48 --- /dev/null +++ b/parser/internal/ingest/detect2016_test.go @@ -0,0 +1,252 @@ +package ingest + +import ( + "testing" + + "github.com/tiennm99/thptqg/parser/internal/reader" +) + +// Ports the 11 tests in parser/src/format_detect_2016.rs, plus a guard per quirk. +// +// Phase 1 settled the Data -> string translation these fixtures need, so no +// guessing: Data::String(s) is s verbatim, Data::Float(8.0) renders "8" (Rust's +// f64 Display drops the .0, matching Go's FormatFloat(v,'f',-1,64)), +// Data::Float(8.5) renders "8.5", and Data::Empty is "" with IsEmpty true. + +// --- header detection --- + +func TestIsHeaderRow2016(t *testing.T) { + if !IsHeaderRow2016(cells("SBD", "HOTEN", "TOAN")) { + t.Error("SBD header not detected") + } + if !IsHeaderRow2016(cells("sobaodanh", "x")) { + t.Error("lowercase SOBAODANH not detected") + } + if IsHeaderRow2016(cells("Nguyen Van A", "01/01/2000")) { + t.Error("data row wrongly detected as header") + } + // The 2016 guard is len < 2, unlike the 2017 check's len < 3. + if IsHeaderRow2016(cells("SBD")) { + t.Error("single-cell row must not be a header") + } + if !IsHeaderRow2016(cells("SBD", "x")) { + t.Error("two-cell row with a header token must be a header") + } +} + +// TestKnownHeadersHasTrailingSpaceToken pins the "SINH " literal. Trimming it +// would silently change which rows count as headers. +func TestKnownHeadersHasTrailingSpaceToken(t *testing.T) { + var found bool + for _, h := range KnownHeaders { + if h == "SINH " { + found = true + } + } + if !found { + t.Error(`KnownHeaders must contain "SINH " WITH its trailing space`) + } + if len(KnownHeaders) != 17 { + t.Errorf("KnownHeaders has %d tokens, want 17", len(KnownHeaders)) + } +} + +// --- format detection --- + +func TestDetectFormatSeparateScores(t *testing.T) { + f := DetectFormat(cells("SBD", "HOTEN", "TOAN", "VAN")) + if f.Kind != FormatSeparateScores { + t.Errorf("Kind = %v, want FormatSeparateScores", f.Kind) + } +} + +func TestDetectFormatMapped(t *testing.T) { + f := DetectFormat(cells("STT", "SOBAODANH", "HO_TEN", "NGAY_SINH", "TEN_CUMTHI", "GIOI_TINH", "DIEM_THI")) + if f.Kind != FormatMapped { + t.Fatalf("Kind = %v, want FormatMapped", f.Kind) + } + if f.Sbd != 1 || f.HoTen != 2 || f.DiemThi != 6 { + t.Errorf("indices sbd=%d ho_ten=%d diem_thi=%d", f.Sbd, f.HoTen, f.DiemThi) + } + if f.NgaySinh == nil || *f.NgaySinh != 3 || f.TenCumThi == nil || *f.TenCumThi != 4 || f.GioiTinh == nil || *f.GioiTinh != 5 { + t.Error("optional indices not resolved") + } +} + +// TestDetectFormatMappedIsOrderIndependent: indices are resolved by name. +func TestDetectFormatMappedIsOrderIndependent(t *testing.T) { + f := DetectFormat(cells("DIEM_THI", "HO_TEN", "SBD")) + if f.Kind != FormatMapped || f.Sbd != 2 || f.DiemThi != 0 || f.HoTen != 1 { + t.Errorf("got %+v", f) + } +} + +// TestDetectFormatMappedHoTenFallback ports the col-1 fallback +// (format_detect_2016.rs:132). +func TestDetectFormatMappedHoTenFallback(t *testing.T) { + f := DetectFormat(cells("SBD", "SOMETHING", "DIEM_THI")) + if f.Kind != FormatMapped { + t.Fatalf("Kind = %v, want FormatMapped", f.Kind) + } + if f.HoTen != 1 { + t.Errorf("ho_ten = %d, want fallback 1", f.HoTen) + } +} + +// TestDetectFormatDefault: a header lacking SBD or DIEM_THI falls back to the +// positional layout. +func TestDetectFormatDefault(t *testing.T) { + f := DetectFormat(cells("A", "B", "C")) + if f.Kind != FormatDefault { + t.Errorf("Kind = %v, want FormatDefault", f.Kind) + } + if f.Sbd != 0 || f.HoTen != 1 || f.DiemThi != 5 { + t.Errorf("default indices wrong: %+v", f) + } +} + +// --- row processing --- + +func TestProcessSeparateScoresRow(t *testing.T) { + // 0=SBD 1=HOTEN 2=TOAN 3=VAN 4=LY 5=HOA 6=SINH 7=SU 8=DIA 9,10=NN 11=NN total + row := cells("1000", "Nguyễn Văn Đức", "8", "7.5", "", "6", "", "5", "", "", "", "9.25") + got := ProcessRow2016(row, Format{Kind: FormatSeparateScores}) + if got == nil { + t.Fatal("row rejected") + } + if got.SoBaoDanh != "1000" || got.HoTenAscii != "nguyen van duc" { + t.Errorf("sbd=%q ascii=%q", got.SoBaoDanh, got.HoTenAscii) + } + want := map[string]float64{"toan": 8, "ngu_van": 7.5, "hoa_hoc": 6, "lich_su": 5, "tieng_anh": 9.25} + if len(got.Scores) != len(want) { + t.Errorf("scores = %v, want %v", got.Scores, want) + } + for k, v := range want { + if got.Scores[k] != v { + t.Errorf("%s = %v, want %v", k, got.Scores[k], v) + } + } + // These columns are structurally unreachable in this layout. + for _, absent := range []string{"tieng_phap", "tieng_duc", "tieng_nhat", "tieng_trung", "khtn", "khxh", "gdcd"} { + if _, ok := got.Scores[absent]; ok { + t.Errorf("%s must be unreachable in separate-scores", absent) + } + } + if got.NgaySinh != nil || got.TenCumThi != nil || got.GioiTinh != nil { + t.Error("ngay_sinh/ten_cum_thi/gioi_tinh are always nil in separate-scores") + } +} + +// TestZeroScoreBecomesNull pins the JS falsy quirk: parseFloat(x) || null means +// a literal 0 is indistinguishable from "no score" (format_detect_2016.rs:165). +func TestZeroScoreBecomesNull(t *testing.T) { + row := cells("1000", "A", "0", "0.0", "1", "", "", "", "", "", "", "") + got := ProcessRow2016(row, Format{Kind: FormatSeparateScores}) + if got == nil { + t.Fatal("row rejected") + } + if _, ok := got.Scores["toan"]; ok { + t.Error(`a "0" score must become NULL, not 0`) + } + if _, ok := got.Scores["ngu_van"]; ok { + t.Error(`a "0.0" score must become NULL, not 0`) + } + if got.Scores["vat_ly"] != 1 { + t.Error("a non-zero score must survive") + } +} + +// TestGenderAllowlist ports format_detect_2016.rs:263-271 — exactly two values. +func TestGenderAllowlist(t *testing.T) { + f := defaultFormat() + for in, want := range map[string]string{"Nam": "Nam", "Nữ": "Nữ"} { + row := cells("1", "A", "", "", in, "") + got := ProcessRow2016(row, f) + if got == nil || got.GioiTinh == nil || *got.GioiTinh != want { + t.Errorf("gioi_tinh for %q not preserved", in) + } + } + for _, in := range []string{"nam", "NAM", "Unknown", "M", "F", "nữ", ""} { + row := cells("1", "A", "", "", in, "") + got := ProcessRow2016(row, f) + if got == nil { + t.Fatalf("row rejected for %q", in) + } + if got.GioiTinh != nil { + t.Errorf("gioi_tinh for %q = %q, want nil", in, *got.GioiTinh) + } + } +} + +// TestLeakedHeaderRowSkipped ports format_detect_2016.rs:244-250. +func TestLeakedHeaderRowSkipped(t *testing.T) { + f := defaultFormat() + if ProcessRow2016(cells("SBD", "HO_TEN", "", "", "", ""), f) != nil { + t.Error("a repeated header row must be skipped") + } + if ProcessRow2016(cells("123", "SOBAODANH", "", "", "", ""), f) != nil { + t.Error("a row whose name cell is a header token must be skipped") + } + if ProcessRow2016(cells("123", "Nguyen Van A", "", "", "", ""), f) == nil { + t.Error("a genuine data row must not be skipped") + } +} + +// TestMappedRowEmptyFieldsRejected: either identity field empty rejects the row. +func TestMappedRowEmptyFieldsRejected(t *testing.T) { + f := defaultFormat() + if ProcessRow2016(cells("", "A", "", "", "", ""), f) != nil { + t.Error("empty sbd must reject") + } + if ProcessRow2016(cells("1", "", "", "", "", ""), f) != nil { + t.Error("empty ho_ten must reject") + } +} + +// TestMappedRowPopulates2016OnlyColumns: this is the only dataset that fills +// ten_cum_thi and gioi_tinh. +func TestMappedRowPopulates2016OnlyColumns(t *testing.T) { + row := cells("123", "Lê Văn Long", "01/01/1998", "Cụm thi số 1", "Nam", "Toán: 7.5 Tiếng Đức: 6") + got := ProcessRow2016(row, defaultFormat()) + if got == nil { + t.Fatal("row rejected") + } + if got.NgaySinh == nil || *got.NgaySinh != "01/01/1998" { + t.Errorf("ngay_sinh = %v", got.NgaySinh) + } + if got.TenCumThi == nil || *got.TenCumThi != "Cụm thi số 1" { + t.Errorf("ten_cum_thi = %v", got.TenCumThi) + } + if got.Scores["toan"] != 7.5 || got.Scores["tieng_duc"] != 6 { + t.Errorf("scores = %v", got.Scores) + } +} + +// TestDefaultIsMappedWithFixedIndices: FormatDefault must not be a separate code +// path (format_detect_2016.rs:295-306). +func TestDefaultIsMappedWithFixedIndices(t *testing.T) { + row := cells("123", "A", "01/01/2000", "Cluster", "Nữ", "Toán: 5") + viaDefault := ProcessRow2016(row, defaultFormat()) + two, three, four := 2, 3, 4 + viaMapped := ProcessRow2016(row, Format{ + Kind: FormatMapped, Sbd: 0, HoTen: 1, + NgaySinh: &two, TenCumThi: &three, GioiTinh: &four, DiemThi: 5, + }) + if viaDefault == nil || viaMapped == nil { + t.Fatal("row rejected") + } + if viaDefault.SoBaoDanh != viaMapped.SoBaoDanh || + *viaDefault.TenCumThi != *viaMapped.TenCumThi || + *viaDefault.GioiTinh != *viaMapped.GioiTinh || + viaDefault.Scores["toan"] != viaMapped.Scores["toan"] { + t.Error("FormatDefault must behave identically to the equivalent FormatMapped") + } +} + +// TestShortRowGuardIsSeparateFromValidation documents that rows under 2 cells +// are dropped by the loop before the counter (main.rs:351-353), not here. +func TestShortRowGuardIsSeparateFromValidation(t *testing.T) { + if got := ProcessRow2016([]reader.Cell{{Str: "123"}}, defaultFormat()); got != nil { + t.Error("a 1-cell row has no name and must be rejected") + } +} diff --git a/parser/internal/ingest/ingest.go b/parser/internal/ingest/ingest.go new file mode 100644 index 0000000..377a1d0 --- /dev/null +++ b/parser/internal/ingest/ingest.go @@ -0,0 +1,243 @@ +// Package ingest owns all dataset policy: which sheets to read, which rows are +// headers, which are blank, and how rows are counted. +// +// The reader deliberately has none of this — it reports every sheet and every +// row exactly as calamine would, which is what made its fidelity independently +// testable. This package ports the build loop in parser/src/main.rs plus the +// header/blank helpers in parser/src/reader.rs. +package ingest + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/reader" + "github.com/tiennm99/thptqg/parser/internal/transform" + "github.com/tiennm99/thptqg/parser/internal/writer" +) + +// IsHeaderRow reports whether row is a header, by matching its uppercased first +// cell against the configured tokens (reader.rs:28-34). +// +// Rows shorter than 3 cells are never headers (reader.rs:29) — a 1- or 2-cell +// row is a stray fragment, not a real header. +func IsHeaderRow(row []reader.Cell, tokens []string) bool { + if len(row) < 3 { + return false + } + first := strings.ToUpper(strings.TrimSpace(row[0].Str)) + for _, t := range tokens { + if strings.ToUpper(t) == first { + return true + } + } + return false +} + +// IsAllBlank reports whether every cell is empty or whitespace-only +// (reader.rs:40-43). +// +// Compares on Str only. Cell.IsEmpty is diagnostic: calamine distinguishes +// Data::Empty from an empty string cell, but both render "" and both count as +// blank here, so branching on the flag would invent a distinction the Rust +// original never acts on. +func IsAllBlank(row []reader.Cell) bool { + for _, c := range row { + if strings.TrimSpace(c.Str) != "" { + return false + } + } + return true +} + +// InputFiles lists a dataset directory's spreadsheets, sorted. +// +// The sort is load-bearing, not cosmetic: INSERT OR REPLACE is last-wins, so +// file order decides which row survives a duplicate SBD. Rust collects read_dir +// then calls files.sort() (main.rs:82-97) — a bytewise sort on the full path. +func InputFiles(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("cannot read input dir %s: %w", dir, err) + } + var out []string + for _, e := range entries { + if e.IsDir() { + continue + } + switch strings.ToLower(filepath.Ext(e.Name())) { + case ".xls", ".xlsx": + out = append(out, filepath.Join(dir, e.Name())) + } + } + sort.Strings(out) + return out, nil +} + +// DatasetLabel derives the stats-wording key from the input directory basename, +// matching main.rs:98-101. +func DatasetLabel(inputDir string) string { + base := filepath.Base(strings.TrimRight(inputDir, string(filepath.Separator))) + if base == "" || base == "." || base == string(filepath.Separator) { + return "data" + } + return base +} + +// RowFn consumes one data row of one sheet, after header skipping. +type RowFn func(sheetIdx int, row []reader.Cell) + +// ProcessFile applies sheet selection and per-sheet header skipping, invoking fn +// for every remaining row — the port of reader.rs:54-105. +// +// The header check is per SHEET, not per file: first_row resets inside the sheet +// loop (reader.rs:91), so a workbook whose second sheet repeats the header has +// it skipped there too. +func ProcessFile(path string, cfg *config.DatasetConfig, fn RowFn) error { + wb, err := reader.Open(path) + if err != nil { + return err + } + defer wb.Close() + + sheets := wb.Sheets() + if len(sheets) == 0 { + return fmt.Errorf("no sheets in %s", path) + } + if cfg.Reader.SheetMode == config.SheetModeFirst { + sheets = sheets[:1] + } + + for _, sh := range sheets { + firstRow := true + err := wb.EachRow(sh.Index, func(s reader.Sheet, _ int, row []reader.Cell) error { + if firstRow { + firstRow = false + if IsHeaderRow(row, cfg.Header.Tokens) { + return nil + } + } + fn(s.Index, row) + return nil + }) + if err != nil { + return err + } + } + return nil +} + +// Standard runs the fixed-column path for the 2017-family datasets — the port of +// run_build_standard (main.rs:74-199). +func Standard(cfg *config.DatasetConfig, inputDir, outputPath string) error { + files, err := InputFiles(inputDir) + if err != nil { + return err + } + label := DatasetLabel(inputDir) + fmt.Printf("[build] %s/ → %s (%d files)\n", label, outputPath, len(files)) + + db, err := writer.OpenDB(outputPath) + if err != nil { + return err + } + defer db.Close() + + isOld2 := strings.Contains(label, "old2") + stripBlank := cfg.Reader.StripBlankRows + + // One transaction spans the whole dataset directory (main.rs:120,184). + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin: %w", err) + } + ins, err := writer.Prepare(tx) + if err != nil { + tx.Rollback() + return err + } + + var st writer.Stats + for _, file := range files { + base := filepath.Base(file) + var fileRows, fileSkipped, fileErrors uint64 + + procErr := ProcessFile(file, cfg, func(_ int, row []reader.Cell) { + allBlank := IsAllBlank(row) + // 2017-old2: blank rows drop out BEFORE the source-row counter + // (main.rs:134-137, ahead of the increment at :140). + if stripBlank && allBlank { + return + } + st.SourceRows++ + + hoTen, soBaoDanh := "", "" + if cols := cfg.Columns; cols != nil { + hoTen = cellAt(row, cols.HoTen) + soBaoDanh = cellAt(row, cols.SoBaoDanh) + } + + switch transform.ValidateRow(hoTen, soBaoDanh, &cfg.Validation, stripBlank, allBlank) { + case transform.SkipBlankRow: + // Falls through to transform and insert, matching main.rs:150. + // Unreachable here: BlankRow requires stripBlank && allBlank, + // which returned above. Kept so the two call sites with opposite + // outcomes stay visibly distinct. + case transform.SkipNone: + // proceed + default: + fileSkipped++ + return + } + + parsed, err := transform.TransformRow(row, cfg) + if err != nil { + fileErrors++ + return + } + if err := ins.Insert(parsed); err != nil { + fileErrors++ + // Only the first five insert warnings print (main.rs:164). + if st.Errors+fileErrors <= 5 { + fmt.Fprintf(os.Stderr, " [warn] %s: %v\n", base, err) + } + return + } + fileRows++ + }) + if procErr != nil { + // A file that cannot be read is logged and counted, never fatal + // (main.rs:171-177) — one corrupt file must not abandon the batch. + fmt.Fprintf(os.Stderr, " [error] %s: %v\n", base, procErr) + fileErrors++ + } + + st.Skipped += fileSkipped + st.Errors += fileErrors + fmt.Printf(" %s: %d rows\n", base, fileRows) + } + + if err := ins.Close(); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + + // VACUUM only after COMMIT — SQLite refuses it inside a transaction. + return writer.Finish(db, outputPath, st, label, isOld2) +} + +// cellAt returns the trimmed cell at idx, or "" when out of range — the +// unwrap_or_default() behaviour of transform.rs:163-167. +func cellAt(row []reader.Cell, idx int) string { + if idx < 0 || idx >= len(row) { + return "" + } + return strings.TrimSpace(row[idx].Str) +} diff --git a/parser/internal/ingest/ingest_test.go b/parser/internal/ingest/ingest_test.go new file mode 100644 index 0000000..22a4e06 --- /dev/null +++ b/parser/internal/ingest/ingest_test.go @@ -0,0 +1,131 @@ +package ingest + +import ( + "os" + "path/filepath" + "testing" + + "github.com/tiennm99/thptqg/parser/internal/reader" +) + +// Ports the 7 tests in parser/src/reader.rs:116-197. They were listed under +// Phase 1 originally, which was wrong: they exercise header and blank-row +// policy, which lives here rather than in the reader package. + +func cells(vals ...string) []reader.Cell { + out := make([]reader.Cell, len(vals)) + for i, v := range vals { + out[i] = reader.Cell{Str: v, IsEmpty: v == ""} + } + return out +} + +var stdTokens = []string{"HO_TEN", "HỌ TÊN", "STT"} + +// header_detects_ho_ten (reader.rs:128) +func TestHeaderDetectsHoTen(t *testing.T) { + if !IsHeaderRow(cells("HO_TEN", "NGAY_SINH", "SBD"), stdTokens) { + t.Error("HO_TEN header not detected") + } +} + +// header_detects_stt (reader.rs:139) +func TestHeaderDetectsStt(t *testing.T) { + if !IsHeaderRow(cells("STT", "B", "C"), stdTokens) { + t.Error("STT header not detected") + } +} + +// header_detects_ho_ten_unicode (reader.rs:150) +func TestHeaderDetectsHoTenUnicode(t *testing.T) { + if !IsHeaderRow(cells("HỌ TÊN", "B", "C"), stdTokens) { + t.Error("HỌ TÊN header not detected") + } +} + +// header_rejects_data_row (reader.rs:161) +func TestHeaderRejectsDataRow(t *testing.T) { + if IsHeaderRow(cells("Nguyen Van A", "01/01/2000", "12345678"), stdTokens) { + t.Error("data row wrongly detected as header") + } +} + +// header_rejects_short_row (reader.rs:172) — the <3 cell guard. +func TestHeaderRejectsShortRow(t *testing.T) { + if IsHeaderRow(cells("HO_TEN", ""), stdTokens) { + t.Error("a 2-cell row must never be a header, even with a matching token") + } +} + +// header_case_insensitive (reader.rs:179) +func TestHeaderCaseInsensitive(t *testing.T) { + if !IsHeaderRow(cells("ho_ten", "B", "C"), stdTokens) { + t.Error("lowercase header not detected") + } +} + +// blank_row_detection (reader.rs:190) — note the third cell is an empty *string* +// cell, not Data::Empty, and must still count as blank. +func TestBlankRowDetection(t *testing.T) { + blank := []reader.Cell{{IsEmpty: true}, {IsEmpty: true}, {Str: ""}} + if !IsAllBlank(blank) { + t.Error("row of empty cells should be blank") + } + if IsAllBlank(cells("Nguyen", "", "")) { + t.Error("row with content should not be blank") + } +} + +// TestIsAllBlankIgnoresIsEmptyFlag pins the rule that blankness is decided by +// the rendered string, never by Cell.IsEmpty — calamine emits empty-but-not-Empty +// cells, and treating the flag as authoritative would diverge. +func TestIsAllBlankIgnoresIsEmptyFlag(t *testing.T) { + // Content present but IsEmpty wrongly set: still not blank. + if IsAllBlank([]reader.Cell{{Str: "x", IsEmpty: true}}) { + t.Error("a cell with content must not be blank regardless of IsEmpty") + } + // Whitespace only: blank, matching Rust's trim().is_empty(). + if !IsAllBlank([]reader.Cell{{Str: " "}, {Str: "\t"}}) { + t.Error("whitespace-only cells should be blank") + } +} + +// TestDatasetLabel covers the basename derivation that drives the stats wording. +func TestDatasetLabel(t *testing.T) { + for in, want := range map[string]string{ + "data/2017": "2017", + "data/2017-old2": "2017-old2", + "data/2017-old2/": "2017-old2", + "/abs/path/2016": "2016", + } { + if got := DatasetLabel(in); got != want { + t.Errorf("DatasetLabel(%q) = %q, want %q", in, got, want) + } + } +} + +// TestInputFilesSortedAndFiltered: the sort decides which duplicate SBD survives +// INSERT OR REPLACE, so it is behaviour, not presentation. +func TestInputFilesSortedAndFiltered(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"b.xlsx", "a.xls", "c.XLSX", "notes.txt", "d.csv"} { + if err := writeEmpty(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + got, err := InputFiles(dir) + if err != nil { + t.Fatal(err) + } + want := []string{"a.xls", "b.xlsx", "c.XLSX"} + if len(got) != len(want) { + t.Fatalf("got %d files %v, want %d", len(got), got, len(want)) + } + for i := range want { + if filepath.Base(got[i]) != want[i] { + t.Errorf("file %d = %q, want %q", i, filepath.Base(got[i]), want[i]) + } + } +} + +func writeEmpty(path string) error { return os.WriteFile(path, nil, 0o644) } diff --git a/parser/internal/reader/fidelity_test.go b/parser/internal/reader/fidelity_test.go new file mode 100644 index 0000000..133f096 --- /dev/null +++ b/parser/internal/reader/fidelity_test.go @@ -0,0 +1,128 @@ +package reader_test + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tiennm99/thptqg/parser/internal/reader" +) + +// TestReaderFidelity asserts the Go reader reproduces calamine byte-for-byte on +// every real input file. +// +// The oracle is a committed SHA-256 per file over a canonical cell dump. The +// dumps themselves are real student names and birthdates, so only the hashes are +// committed — regenerate the dumps from the Rust side on demand +// (parser/examples/dump_cells.rs), which is possible because parser/ still +// builds. +// +// The canonical form carries geometry and rendered cell values. The calamine +// Data variant is excluded on purpose: Data::Empty and Data::String("") both +// render "" and both count as blank in is_all_blank and transform, so the +// distinction cannot reach the database. +// Runs by default so CI and `go test ./...` keep the full guarantee; skipped +// under -short, which is how to iterate without paying ~77s to re-read 418 MB. +func TestReaderFidelity(t *testing.T) { + if testing.Short() { + t.Skip("-short: skipping the 299-file corpus sweep") + } + root := repoRoot(t) + manifest := filepath.Join(root, "parser", "testdata", "reader-fidelity-hashes.tsv") + + f, err := os.Open(manifest) + if err != nil { + t.Fatalf("open manifest: %v", err) + } + defer f.Close() + + var checked int + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + if line == "" || strings.HasPrefix(line, "#") { + continue + } + rel, want, ok := strings.Cut(line, "\t") + if !ok { + t.Fatalf("malformed manifest line: %q", line) + } + path := filepath.Join(root, rel) + if _, err := os.Stat(path); err != nil { + t.Skipf("input data not present (%s); skipping fidelity suite", rel) + } + + checked++ + t.Run(rel, func(t *testing.T) { + t.Parallel() + got, err := canonicalHash(path) + if err != nil { + t.Fatalf("hash %s: %v", rel, err) + } + if got != want { + t.Errorf("cell dump diverges from calamine\n want %s\n got %s", want, got) + } + }) + } + if err := sc.Err(); err != nil { + t.Fatalf("read manifest: %v", err) + } + if checked == 0 { + t.Fatal("manifest contained no entries") + } +} + +// canonicalHash renders one file in the canonical form and hashes it. Kept +// byte-identical to the awk canonicalisation used to build the manifest. +func canonicalHash(path string) (string, error) { + wb, err := reader.Open(path) + if err != nil { + return "", err + } + defer wb.Close() + + h := sha256.New() + sheets := wb.Sheets() + fmt.Fprintf(h, "SHEETCOUNT\t%d\n", len(sheets)) + for _, sh := range sheets { + fmt.Fprintf(h, "SHEET\t%d\t%s\t%d\t%d\n", sh.Index, escape(sh.Name), sh.Height, sh.Width) + err := wb.EachRow(sh.Index, func(s reader.Sheet, rowIdx int, row []reader.Cell) error { + fmt.Fprintf(h, "ROW\t%d\t%d\t%d\n", s.Index, rowIdx, len(row)) + for c, cell := range row { + fmt.Fprintf(h, "CELL\t%d\t%d\t%d\t%s\n", s.Index, rowIdx, c, escape(cell.Str)) + } + return nil + }) + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func escape(s string) string { + return strings.NewReplacer("\\", `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`).Replace(s) +} + +// repoRoot walks up from the test's working directory to the directory holding +// the data/ corpus. +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + for i := 0; i < 6; i++ { + if _, err := os.Stat(filepath.Join(dir, "data")); err == nil { + return dir + } + dir = filepath.Dir(dir) + } + t.Fatal("could not locate repo root (no data/ directory found)") + return "" +} diff --git a/parser/internal/reader/reader.go b/parser/internal/reader/reader.go new file mode 100644 index 0000000..79f3fda --- /dev/null +++ b/parser/internal/reader/reader.go @@ -0,0 +1,79 @@ +// Package reader wraps the two spreadsheet libraries behind one streaming +// contract, mirroring parser/src/reader.rs (which wraps calamine's +// open_workbook_auto for both formats). +// +// The contract is deliberately row-streaming rather than whole-workbook +// materialising: data/2017/ha-noi.xls alone holds 72k rows across two sheets, +// and the Rust original keeps at most one sheet range live at a time. +package reader + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Cell is one spreadsheet cell rendered the way calamine's Data::to_string() +// renders it. +// +// IsEmpty tracks calamine's Data::Empty variant separately from a string cell +// that happens to be empty. Rust distinguishes them at reader.rs:42, but both +// render "" and both count as blank in is_all_blank and in transform, so the +// flag is diagnostic only — never compare on it. +type Cell struct { + Str string + IsEmpty bool +} + +// Sheet carries a sheet's identity and geometry. Height and Width describe the +// used range, matching calamine's Range::height()/width(); every used range in +// the corpus starts at (0,0), verified across all 299 files. +type Sheet struct { + Index int + Name string + Height int + Width int +} + +// RowFunc receives each row of each sheet. Rows are padded to the sheet's used +// width — width is load-bearing because every column read downstream is +// positional with an unwrap_or_default() equivalent, so a short row silently +// NULLs its tail columns. +type RowFunc func(sheet Sheet, rowIdx int, row []Cell) error + +// Workbook is one opened spreadsheet. +type Workbook interface { + // Sheets returns sheet identity and geometry in workbook order. + Sheets() []Sheet + // EachRow streams every row of the given sheet in order. + EachRow(sheetIdx int, fn RowFunc) error + Close() error +} + +// Open dispatches on file extension, mirroring calamine's open_workbook_auto. +func Open(path string) (Workbook, error) { + switch strings.ToLower(filepath.Ext(path)) { + case ".xls": + return openXLS(path) + case ".xlsx", ".xlsm": + return openXLSX(path) + default: + return nil, fmt.Errorf("unsupported extension: %s", path) + } +} + +// padRow extends row to width with empty cells, and truncates if longer. +func padRow(row []Cell, width int) []Cell { + if len(row) == width { + return row + } + if len(row) > width { + return row[:width] + } + out := make([]Cell, width) + copy(out, row) + for i := len(row); i < width; i++ { + out[i] = Cell{IsEmpty: true} + } + return out +} diff --git a/parser/internal/reader/xls.go b/parser/internal/reader/xls.go new file mode 100644 index 0000000..8f15c06 --- /dev/null +++ b/parser/internal/reader/xls.go @@ -0,0 +1,119 @@ +package reader + +import ( + "fmt" + + "github.com/pbnjay/grate" + // Registers the BIFF backend with grate.Open. + _ "github.com/pbnjay/grate/xls" +) + +// xlsWorkbook reads legacy BIFF through pbnjay/grate. +// +// grate replaced extrame/xls, which was measured against calamine ground truth +// and found to corrupt 69% of cells and drop a further 28%: undecoded UTF-16LE +// and BIFF record framing leaked into cell values, content moved between rows +// and columns, and tail rows came back blank. That was charset-independent and +// unfixable from the outside. grate reproduces calamine exactly on the same +// files. +// +// The one normalisation grate needs is trailing blank rows: it yields rows past +// the end of calamine's used range (one for a populated sheet, two for an empty +// one), so trailing all-blank rows are trimmed. Note this is the opposite of +// the xlsx path, where excelize already trims and calamine keeps a 1x1 empty +// range — in both cases the rule is "match calamine's used range". +type xlsWorkbook struct { + sheets []Sheet + rows [][][]Cell +} + +func openXLS(path string) (Workbook, error) { + wb, err := grate.Open(path) + if err != nil { + return nil, fmt.Errorf("grate open %s: %w", path, err) + } + defer wb.Close() + + names, err := wb.List() + if err != nil { + return nil, fmt.Errorf("grate list %s: %w", path, err) + } + + out := &xlsWorkbook{} + for idx, name := range names { + sh, err := wb.Get(name) + if err != nil { + return nil, fmt.Errorf("grate get %s/%s: %w", path, name, err) + } + + var raw [][]string + for sh.Next() { + row := sh.Strings() + cp := make([]string, len(row)) + for i, v := range row { + cp[i] = demergeMarker(v) + } + raw = append(raw, cp) + } + + // Trim to calamine's used range. + height := len(raw) + for height > 0 && rowAllBlank(raw[height-1]) { + height-- + } + raw = raw[:height] + + width := 0 + for _, r := range raw { + if len(r) > width { + width = len(r) + } + } + + cells := make([][]Cell, len(raw)) + for i, r := range raw { + row := make([]Cell, len(r)) + for j, v := range r { + row[j] = Cell{Str: v, IsEmpty: v == ""} + } + cells[i] = padRow(row, width) + } + + out.sheets = append(out.sheets, Sheet{Index: idx, Name: name, Height: height, Width: width}) + out.rows = append(out.rows, cells) + } + return out, nil +} + +// demergeMarker blanks grate's merged-cell continuation markers. +// +// grate fills the cells covered by a merge with sentinel runes; calamine +// reports them as empty. In this corpus they occur only in the merged title +// block of the 2016 spreadsheets (19 cells in rows 0-2 of one file). Only an +// exact whole-value match is blanked, so a real cell that merely contains an +// arrow is untouched. +func demergeMarker(v string) string { + switch v { + case grate.ContinueColumnMerged, grate.EndColumnMerged, + grate.ContinueRowMerged, grate.EndRowMerged: + return "" + } + return v +} + +func (w *xlsWorkbook) Sheets() []Sheet { return w.sheets } + +func (w *xlsWorkbook) EachRow(sheetIdx int, fn RowFunc) error { + if sheetIdx < 0 || sheetIdx >= len(w.sheets) { + return fmt.Errorf("sheet index %d out of range", sheetIdx) + } + sh := w.sheets[sheetIdx] + for i, row := range w.rows[sheetIdx] { + if err := fn(sh, i, row); err != nil { + return err + } + } + return nil +} + +func (w *xlsWorkbook) Close() error { return nil } diff --git a/parser/internal/reader/xlsx.go b/parser/internal/reader/xlsx.go new file mode 100644 index 0000000..399901c --- /dev/null +++ b/parser/internal/reader/xlsx.go @@ -0,0 +1,105 @@ +package reader + +import ( + "fmt" + + "github.com/xuri/excelize/v2" +) + +// xlsxWorkbook reads OOXML through excelize. +// +// Two excelize behaviours must be corrected to match calamine: +// +// 1. GetRows applies the cell number format by default, while calamine renders +// the underlying value. RawCellValue: true disables that. +// 2. GetRows trims trailing blank cells, so rows are ragged; calamine returns a +// rectangular used range. Rows are padded back out to the sheet width. +type xlsxWorkbook struct { + f *excelize.File + sheets []Sheet + rows [][][]Cell // [sheetIdx][rowIdx][colIdx] +} + +func openXLSX(path string) (Workbook, error) { + f, err := excelize.OpenFile(path) + if err != nil { + return nil, fmt.Errorf("excelize open %s: %w", path, err) + } + + wb := &xlsxWorkbook{f: f} + crFixups := buildCRFixups(path) + for idx, name := range f.GetSheetList() { + raw, err := f.GetRows(name, excelize.Options{RawCellValue: true}) + if err != nil { + f.Close() + return nil, fmt.Errorf("excelize GetRows %s/%s: %w", path, name, err) + } + + // Do NOT trim trailing blank rows: calamine's used range keeps them, and + // excelize's GetRows already drops trailing fully-empty rows itself. + // + // One correction is needed. 63 sheets in 2017-old and 53 in 2017-old2 + // hold a single empty shared-string cell at A1; calamine reports those + // as a 1x1 range, while GetRows returns nothing. A genuinely empty sheet + // (230 of them in 2016) is height 0 on both sides. GetCellType tells the + // two apart: the empty-shared-string cell exists in the XML and types as + // CellTypeSharedString, an absent cell types as CellTypeUnset. + if len(raw) == 0 { + if t, terr := f.GetCellType(name, "A1"); terr == nil && t != excelize.CellTypeUnset { + raw = [][]string{{""}} + } + } + height := len(raw) + + width := 0 + for _, r := range raw { + if len(r) > width { + width = len(r) + } + } + + cells := make([][]Cell, len(raw)) + for i, r := range raw { + row := make([]Cell, len(r)) + for j, v := range r { + if fixed, ok := crFixups[v]; ok { + v = fixed + } else { + v = normalizeNumeric(f, name, j, i, v) + } + row[j] = Cell{Str: v, IsEmpty: v == ""} + } + cells[i] = padRow(row, width) + } + + wb.sheets = append(wb.sheets, Sheet{Index: idx, Name: name, Height: height, Width: width}) + wb.rows = append(wb.rows, cells) + } + return wb, nil +} + +func rowAllBlank(r []string) bool { + for _, v := range r { + if v != "" { + return false + } + } + return true +} + +func (w *xlsxWorkbook) Sheets() []Sheet { return w.sheets } + +func (w *xlsxWorkbook) EachRow(sheetIdx int, fn RowFunc) error { + if sheetIdx < 0 || sheetIdx >= len(w.sheets) { + return fmt.Errorf("sheet index %d out of range", sheetIdx) + } + sh := w.sheets[sheetIdx] + for i, row := range w.rows[sheetIdx] { + if err := fn(sh, i, row); err != nil { + return err + } + } + return nil +} + +func (w *xlsxWorkbook) Close() error { return w.f.Close() } diff --git a/parser/internal/reader/xlsx_fixups.go b/parser/internal/reader/xlsx_fixups.go new file mode 100644 index 0000000..d66bc25 --- /dev/null +++ b/parser/internal/reader/xlsx_fixups.go @@ -0,0 +1,155 @@ +package reader + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "io" + "strconv" + + "github.com/xuri/excelize/v2" +) + +// normalizeNumeric reproduces calamine's rendering of a numeric cell. +// +// calamine parses a numeric cell to f64 and renders it with Rust's f64 Display, +// so the stored literal "6.0" becomes "6". RawCellValue hands back the literal. +// +// The cell type must be consulted, not guessed: "01063476", "6.00" and "NAN" are +// all shared strings that survive ParseFloat, and renumbering them would drop a +// leading zero, drop a trailing zero, or recase NaN. GetCellType is only called +// when re-rendering would actually change the text, which keeps it off the hot +// path for the ~99% of cells that are already canonical or plainly non-numeric. +func normalizeNumeric(f *excelize.File, sheet string, col, row int, v string) string { + if v == "" { + return v + } + fv, err := strconv.ParseFloat(v, 64) + if err != nil { + return v + } + out := strconv.FormatFloat(fv, 'f', -1, 64) + if out == v { + return v + } + axis, err := excelize.CoordinatesToCellName(col+1, row+1) + if err != nil { + return v + } + // OOXML omits the t attribute on numeric cells, and excelize has no map + // entry for an empty t, so a plain number reports CellTypeUnset rather than + // CellTypeNumber. Unset is only reachable here for a cell that exists and + // parsed as a float — an absent cell is "" and returned above — so both + // values mean "numeric". Shared strings report CellTypeSharedString and are + // left alone, which is what protects "01063476", "6.00" and "NAN". + if t, err := f.GetCellType(sheet, axis); err == nil && + (t == excelize.CellTypeNumber || t == excelize.CellTypeUnset) { + return out + } + return v +} + +// buildCRFixups maps a shared string's line-ending-normalised form back to its +// raw form, for the strings that contain a carriage return. +// +// Go's encoding/xml performs the line-ending normalisation the XML 1.0 spec +// mandates (CRLF and lone CR both become LF), so excelize returns "a\nb" where +// calamine — which reads the raw bytes — returns "a\r\nb". That difference +// reaches the database: in one 2016 file it affects 2,233 TEN_CUMTHI values, +// which populate the ten_cum_thi column. +// +// The trick is that character references are exempt from that normalisation, so +// rewriting literal CR bytes to before decoding round-trips them intact. +// +// Returns nil when the file has no CR at all, which is the common case. +func buildCRFixups(path string) map[string]string { + zr, err := zip.OpenReader(path) + if err != nil { + return nil + } + defer zr.Close() + + var entry *zip.File + for _, zf := range zr.File { + if zf.Name == "xl/sharedStrings.xml" { + entry = zf + break + } + } + if entry == nil { + return nil + } + rc, err := entry.Open() + if err != nil { + return nil + } + defer rc.Close() + data, err := io.ReadAll(rc) + if err != nil || !bytes.ContainsRune(data, '\r') { + return nil + } + data = bytes.ReplaceAll(data, []byte{'\r'}, []byte(" ")) + + fixups := make(map[string]string) + dec := xml.NewDecoder(bytes.NewReader(data)) + var cur bytes.Buffer + inSI, inT := false, false + for { + tok, err := dec.Token() + if err != nil { + break + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "si": + inSI, cur = true, bytes.Buffer{} + case "t": + inT = true + } + case xml.CharData: + if inSI && inT { + cur.Write(t) + } + case xml.EndElement: + switch t.Name.Local { + case "t": + inT = false + case "si": + if inSI { + raw := cur.String() + if norm := normalizeEOL(raw); norm != raw { + fixups[norm] = raw + } + } + inSI = false + } + } + } + if len(fixups) == 0 { + return nil + } + return fixups +} + +// normalizeEOL applies XML 1.0 line-ending normalisation: CRLF and lone CR +// both collapse to LF. This is what encoding/xml does to the raw bytes, so it +// reproduces the text excelize hands back. +func normalizeEOL(s string) string { + if !bytes.ContainsRune([]byte(s), '\r') { + return s + } + var b bytes.Buffer + b.Grow(len(s)) + for i := 0; i < len(s); i++ { + if s[i] == '\r' { + if i+1 < len(s) && s[i+1] == '\n' { + continue // CRLF: the LF is emitted on the next pass + } + b.WriteByte('\n') // lone CR + continue + } + b.WriteByte(s[i]) + } + return b.String() +} diff --git a/parser/internal/schema/schema.go b/parser/internal/schema/schema.go new file mode 100644 index 0000000..3e7acc7 --- /dev/null +++ b/parser/internal/schema/schema.go @@ -0,0 +1,144 @@ +// Package schema is the single source of truth for the SQL shape of every +// dataset — a direct port of parser/src/schema.rs. +// +// All four datasets (2016, 2017, 2017-old, 2017-old2) write into the same +// 22-column student table. Columns a dataset has no data for bind NULL. +// +// Column provenance: +// +// ten_cum_thi, gioi_tinh, tieng_duc, tieng_nhat -> 2016 only +// khtn, khxh, gdcd, tieng_nga -> 2017 datasets only +// everything else -> both +// +// Before this consolidation the DDL, the INSERT and the subject regexes were +// duplicated across four TOML configs, which is how the 2016 and 2017 schemas +// drifted apart. The configs now carry only per-dataset parse rules. +package schema + +import "regexp" + +// DDL is executed verbatim after the output database is (re)created. +// +// idx_ten_cum_thi is partial, so it holds zero entries on the three 2017 +// datasets — where the column is always NULL — while staying useful for the +// 2016 cluster-grouping queries. Partial indexes are SQLite-specific. +// +// Byte-identical to parser/src/schema.rs:26-54; TestDDLMatchesRust enforces it. +const DDL = ` +CREATE TABLE student ( + so_bao_danh TEXT PRIMARY KEY, + ho_ten TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + ngay_sinh TEXT, + ten_cum_thi TEXT, + gioi_tinh TEXT, + toan REAL, + ngu_van REAL, + vat_ly REAL, + hoa_hoc REAL, + sinh_hoc REAL, + khtn REAL, + lich_su REAL, + dia_ly REAL, + gdcd REAL, + khxh REAL, + tieng_anh REAL, + tieng_phap REAL, + tieng_nga REAL, + tieng_duc REAL, + tieng_nhat REAL, + tieng_trung REAL +); +CREATE INDEX idx_ho_ten ON student(ho_ten); +CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); +CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; +` + +// IdentityFields are the identity columns, in INSERT parameter order. +var IdentityFields = []string{ + "so_bao_danh", + "ho_ten", + "ho_ten_ascii", + "ngay_sinh", + "ten_cum_thi", + "gioi_tinh", +} + +// ScoreFields are the subject columns, in INSERT parameter order. Bound NULL +// when a row has no score for that subject. +var ScoreFields = []string{ + "toan", + "ngu_van", + "vat_ly", + "hoa_hoc", + "sinh_hoc", + "khtn", + "lich_su", + "dia_ly", + "gdcd", + "khxh", + "tieng_anh", + "tieng_phap", + "tieng_nga", + "tieng_duc", + "tieng_nhat", + "tieng_trung", +} + +// ParamCount is the total bound parameters per row. +const ParamCount = 22 + +// InsertSQL is a positional INSERT matching IdentityFields then ScoreFields. +// +// OR REPLACE is a behavioural contract, not an optimisation: a repeated SBD +// overwrites the earlier row rather than aborting the transaction, so the last +// file to supply a duplicate wins. +const InsertSQL = ` +INSERT OR REPLACE INTO student + (so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, gioi_tinh, + toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn, + lich_su, dia_ly, gdcd, khxh, + tieng_anh, tieng_phap, tieng_nga, tieng_duc, tieng_nhat, tieng_trung) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +` + +// scorePatternSources holds the regex per subject, applied to the DIEM_THI cell +// text. Copied verbatim from parser/src/schema.rs:126-143 — the literals contain +// Vietnamese text and must never be retyped. +// +// Every pattern runs against every dataset. A subject absent from a given exam +// year simply never matches and stays NULL: 2016 files contain no "KHTN:" or +// "Tiếng Nga:" tokens, and 2017 files contain no "Tiếng Đức:" or "Tiếng Nhật:". +// +// Go's regexp and Rust's regex crate are both RE2, and these patterns use no +// backreferences, lookaround or Unicode classes, so they port with zero risk. +var scorePatternSources = map[string]string{ + "toan": `Toán:\s*(\d+(?:\.\d+)?)`, + "ngu_van": `Ngữ văn:\s*(\d+(?:\.\d+)?)`, + "vat_ly": `Vật lí:\s*(\d+(?:\.\d+)?)`, + "hoa_hoc": `Hóa học:\s*(\d+(?:\.\d+)?)`, + "sinh_hoc": `Sinh học:\s*(\d+(?:\.\d+)?)`, + "khtn": `KHTN:\s*(\d+(?:\.\d+)?)`, + "lich_su": `Lịch sử:\s*(\d+(?:\.\d+)?)`, + "dia_ly": `Địa lí:\s*(\d+(?:\.\d+)?)`, + "gdcd": `GDCD:\s*(\d+(?:\.\d+)?)`, + "khxh": `KHXH:\s*(\d+(?:\.\d+)?)`, + "tieng_anh": `Tiếng Anh:\s*(\d+(?:\.\d+)?)`, + "tieng_phap": `Tiếng Pháp:\s*(\d+(?:\.\d+)?)`, + "tieng_nga": `Tiếng Nga:\s*(\d+(?:\.\d+)?)`, + "tieng_duc": `Tiếng Đức:\s*(\d+(?:\.\d+)?)`, + "tieng_nhat": `Tiếng Nhật:\s*(\d+(?:\.\d+)?)`, + "tieng_trung": `Tiếng Trung:\s*(\d+(?:\.\d+)?)`, +} + +// ScorePatterns holds the compiled subject regexes, compiled once at init. +// Rust compiles them once per run in CompiledPatterns::new; a package-level map +// is the equivalent for a single-threaded CLI. +var ScorePatterns = func() map[string]*regexp.Regexp { + out := make(map[string]*regexp.Regexp, len(scorePatternSources)) + for field, src := range scorePatternSources { + out[field] = regexp.MustCompile(src) + } + return out +}() diff --git a/parser/internal/schema/schema_test.go b/parser/internal/schema/schema_test.go new file mode 100644 index 0000000..f657e8c --- /dev/null +++ b/parser/internal/schema/schema_test.go @@ -0,0 +1,150 @@ +package schema + +import ( + "strings" + "testing" +) + +// Ports the four tests in parser/src/schema.rs:149-213. Their purpose is to stop +// the DDL, the INSERT column list and the field-order constants from drifting +// apart — a drift that silently lands values in the wrong columns. + +// TestInsertMatchesFieldOrder ports insert_matches_field_order (schema.rs:156). +func TestInsertMatchesFieldOrder(t *testing.T) { + if ParamCount != 22 { + t.Errorf("ParamCount = %d, want 22", ParamCount) + } + if got := strings.Count(InsertSQL, "?"); got != ParamCount { + t.Errorf("INSERT placeholders = %d, want %d", got, ParamCount) + } + + open := strings.Index(InsertSQL, "(") + closeIdx := strings.Index(InsertSQL, ")") + if open < 0 || closeIdx < 0 { + t.Fatal("INSERT must contain a column list") + } + var listed []string + for _, c := range strings.Split(InsertSQL[open+1:closeIdx], ",") { + if c = strings.TrimSpace(c); c != "" { + listed = append(listed, c) + } + } + + want := append(append([]string{}, IdentityFields...), ScoreFields...) + if len(listed) != len(want) { + t.Fatalf("INSERT lists %d columns, want %d", len(listed), len(want)) + } + for i := range want { + if listed[i] != want[i] { + t.Errorf("column %d: INSERT has %q, field order has %q", i, listed[i], want[i]) + } + } +} + +// TestScorePatternsCoverScoreFields ports score_patterns_cover_score_fields +// (schema.rs:183). +func TestScorePatternsCoverScoreFields(t *testing.T) { + if len(ScorePatterns) != len(ScoreFields) { + t.Fatalf("%d patterns for %d score columns", len(ScorePatterns), len(ScoreFields)) + } + inFields := make(map[string]bool, len(ScoreFields)) + for _, f := range ScoreFields { + inFields[f] = true + } + for field := range ScorePatterns { + if !inFields[field] { + t.Errorf("pattern %q has no column", field) + } + } + for _, field := range ScoreFields { + if _, ok := ScorePatterns[field]; !ok { + t.Errorf("column %q has no pattern", field) + } + } +} + +// TestDDLColumnsMatchInsert ports ddl_columns_match_insert (schema.rs:201). +func TestDDLColumnsMatchInsert(t *testing.T) { + for _, field := range append(append([]string{}, IdentityFields...), ScoreFields...) { + if !strings.Contains(DDL, field) { + t.Errorf("DDL missing column %q", field) + } + } +} + +// TestScorePatternsCompile ports score_patterns_compile (schema.rs:208). +// Compilation happens in the package initialiser, so reaching this point already +// proves it; the explicit checks guard against an empty or partial table. +func TestScorePatternsCompile(t *testing.T) { + for field, re := range ScorePatterns { + if re == nil { + t.Errorf("pattern %q is nil", field) + } + } +} + +// TestDDLMatchesRust asserts the DDL is byte-identical to parser/src/schema.rs. +// Anything less and the two parsers can produce structurally different databases +// while every row-level check still passes. +func TestDDLMatchesRust(t *testing.T) { + const want = ` +CREATE TABLE student ( + so_bao_danh TEXT PRIMARY KEY, + ho_ten TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + ngay_sinh TEXT, + ten_cum_thi TEXT, + gioi_tinh TEXT, + toan REAL, + ngu_van REAL, + vat_ly REAL, + hoa_hoc REAL, + sinh_hoc REAL, + khtn REAL, + lich_su REAL, + dia_ly REAL, + gdcd REAL, + khxh REAL, + tieng_anh REAL, + tieng_phap REAL, + tieng_nga REAL, + tieng_duc REAL, + tieng_nhat REAL, + tieng_trung REAL +); +CREATE INDEX idx_ho_ten ON student(ho_ten); +CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); +CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; +` + if DDL != want { + t.Errorf("DDL diverges from parser/src/schema.rs:26-54\n--- got ---\n%s\n--- want ---\n%s", DDL, want) + } +} + +// TestScorePatternsMatchScores exercises each pattern against the shape the +// DIEM_THI cell actually carries, including the wide runs of spaces seen in the +// real corpus. +func TestScorePatternsMatchScores(t *testing.T) { + const cell = "Toán: 8.50 Ngữ văn: 7.00 Tiếng Đức: 9 KHXH: 5.58 " + cases := map[string]string{ + "toan": "8.50", + "ngu_van": "7.00", + "tieng_duc": "9", + "khxh": "5.58", + "tieng_nhat": "", // absent from the cell -> no match + } + for field, want := range cases { + re, ok := ScorePatterns[field] + if !ok { + t.Fatalf("no pattern for %q", field) + } + m := re.FindStringSubmatch(cell) + got := "" + if m != nil { + got = m[1] + } + if got != want { + t.Errorf("%s: matched %q, want %q", field, got, want) + } + } +} diff --git a/parser/internal/sqlitedb/sqlitedb.go b/parser/internal/sqlitedb/sqlitedb.go new file mode 100644 index 0000000..98ed4d6 --- /dev/null +++ b/parser/internal/sqlitedb/sqlitedb.go @@ -0,0 +1,25 @@ +// Package sqlitedb registers the SQLite driver the parser writes with and names +// it in one place. +// +// modernc.org/sqlite is a pure-Go SQLite, chosen so the whole module stays +// cgo-free — grate, excelize and yaml.v3 are pure Go too, so CI can compile with +// CGO_ENABLED=0 and no C toolchain. +// +// It is a machine-transpiled SQLite rather than the upstream C amalgamation that +// Rust's rusqlite --bundled vendors (libsqlite3-sys 0.30.1). Two things make that +// acceptable: this parser uses only plain SQL — no CTEs, window functions, +// triggers or extensions — and the differential gate compares a full-table +// SHA-256 plus PRAGMA table_info/index_list against live Rust output. +// +// Verified on linux/arm64 with v1.56.0 (SQLite 3.53.3): the full DDL including +// the partial idx_ten_cum_thi index, INSERT OR REPLACE, and VACUUM. +// +// Fallback if the differential gate ever implicates the driver: mattn/go-sqlite3 +// is upstream C at the cost of cgo. +package sqlitedb + +// Registers "sqlite" with database/sql. +import _ "modernc.org/sqlite" + +// DriverName is the database/sql driver name to pass to sql.Open. +const DriverName = "sqlite" diff --git a/parser/internal/transform/ascii_crosscheck_test.go b/parser/internal/transform/ascii_crosscheck_test.go new file mode 100644 index 0000000..7ab5279 --- /dev/null +++ b/parser/internal/transform/ascii_crosscheck_test.go @@ -0,0 +1,65 @@ +package transform_test + +import ( + "database/sql" + "os" + "testing" + + _ "github.com/tiennm99/thptqg/parser/internal/sqlitedb" + "github.com/tiennm99/thptqg/parser/internal/transform" +) + +// TestToAsciiAgainstRustOutput cross-checks ToAscii against Rust on real data. +// +// A Rust-built database is its own oracle: every row carries ho_ten alongside +// the ho_ten_ascii that Rust derived from it, so the whole table is a +// name -> expected-slug corpus far broader than the 20 hand-picked unit cases. +// +// Point it at a Rust-built database: +// +// GO_PARSER_RUST_DB=/tmp/rust-2016.db go test ./internal/transform/ +// +// Skips when unset, so the default suite stays hermetic. +func TestToAsciiAgainstRustOutput(t *testing.T) { + path := os.Getenv("GO_PARSER_RUST_DB") + if path == "" { + t.Skip("GO_PARSER_RUST_DB not set; skipping cross-check against Rust output") + } + if _, err := os.Stat(path); err != nil { + t.Skipf("GO_PARSER_RUST_DB=%s not readable: %v", path, err) + } + + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer db.Close() + + rows, err := db.Query("SELECT ho_ten, ho_ten_ascii FROM student") + if err != nil { + t.Fatalf("query: %v", err) + } + defer rows.Close() + + var checked, bad int + for rows.Next() { + var name, rustAscii string + if err := rows.Scan(&name, &rustAscii); err != nil { + t.Fatalf("scan: %v", err) + } + checked++ + if got := transform.ToAscii(name); got != rustAscii { + bad++ + if bad <= 5 { + t.Errorf("ToAscii(%q)\n rust = %q\n go = %q", name, rustAscii, got) + } + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate: %v", err) + } + if checked == 0 { + t.Fatal("database contained no rows") + } + t.Logf("compared %d real names, %d mismatches", checked, bad) +} diff --git a/parser/internal/transform/transform.go b/parser/internal/transform/transform.go new file mode 100644 index 0000000..92af37b --- /dev/null +++ b/parser/internal/transform/transform.go @@ -0,0 +1,223 @@ +// Package transform performs row transformation: ASCII normalisation, score +// regex parsing, and validation — a port of parser/src/transform.rs. +// +// ToAscii replicates build-lib.js toAscii exactly: +// +// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase() +package transform + +import ( + "errors" + "math" + "strconv" + "strings" + + "golang.org/x/text/unicode/norm" + + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/reader" + "github.com/tiennm99/thptqg/parser/internal/schema" +) + +// ToAscii normalises a Vietnamese name to an ASCII slug. +// +// 1. NFD decompose (splits base + combining diacritics) +// 2. Drop combining marks in U+0300..U+036F +// 3. Replace đ/Đ with d (NFD does not decompose them) +// 4. Lowercase +// +// Step 2 filters a LITERAL CODEPOINT RANGE, not a Unicode category. The inline +// comment at transform.rs:53 says "Unicode category M", but the code at :56 is +// the specification and it checks '\u{0300}'..='\u{036f}'. unicode.Is(unicode.Mn, r) +// is strictly broader and would strip marks Rust keeps, silently changing +// ho_ten_ascii — the column the site's accent-insensitive search runs on. +// +// Step order matters and mirrors transform.rs:54-63: the đ/Đ replacement happens +// before lowercasing. +func ToAscii(s string) string { + if s == "" { + return "" + } + decomposed := norm.NFD.String(s) + + var b strings.Builder + b.Grow(len(decomposed)) + for _, r := range decomposed { + if r >= 0x0300 && r <= 0x036F { + continue // combining mark, in the range Rust drops + } + switch r { + case 'đ', 'Đ': + b.WriteByte('d') + default: + b.WriteRune(r) + } + } + return strings.ToLower(b.String()) +} + +// ParsedRow is one row ready for insertion. +type ParsedRow struct { + SoBaoDanh string + HoTen string + HoTenAscii string + NgaySinh *string + // TenCumThi is 2016 only: examination cluster name (TEN_CUMTHI column). + TenCumThi *string + // GioiTinh is 2016 only: gender, normalised to "Nam"/"Nữ" or nil. + GioiTinh *string + // Scores maps subject field -> value. Absent subjects are simply missing and + // bind NULL. + Scores map[string]float64 +} + +// SkipReason says why a row was skipped, or SkipNone when it passed. +// +// The distinction is load-bearing for the printed counters, and the two +// non-blank reasons are counted as source rows while BlankRow is not — but note +// that split lives in the CALLER, not here. parser/src/main.rs has two call +// sites with opposite outcomes for BlankRow: at :135-137 it returns before the +// counter at :140, while at :151 it matches Err(BlankRow) => {} and falls +// through to transform and insert. The build loop must reproduce both. +type SkipReason int + +const ( + SkipNone SkipReason = iota + // SkipBlankRow: row is fully blank (2017-old2 only, checked before the + // source-row counter). + SkipBlankRow + // SkipEmptyField: so_bao_danh or ho_ten empty/missing. + SkipEmptyField + // SkipNonNumericSbd: so_bao_danh contains non-digit characters + // (2017-old / 2017-old2 guard). + SkipNonNumericSbd +) + +func (s SkipReason) String() string { + switch s { + case SkipNone: + return "none" + case SkipBlankRow: + return "blank_row" + case SkipEmptyField: + return "empty_field" + case SkipNonNumericSbd: + return "non_numeric_sbd" + } + return "unknown" +} + +// ValidateRow checks a row against the dataset's validation rules. +// +// Signature mirrors transform.rs:101-107, taking stripBlankRows and allBlank +// explicitly; a shorter signature could not express both blank-row paths. +func ValidateRow(hoTen, soBaoDanh string, cfg *config.ValidationCfg, stripBlankRows, allBlank bool) SkipReason { + // 2017-old2: skip fully blank rows BEFORE counting source rows. + if stripBlankRows && allBlank { + return SkipBlankRow + } + if cfg.RequireNonemptySbd && soBaoDanh == "" { + return SkipEmptyField + } + if cfg.RequireNonemptyName && hoTen == "" { + return SkipEmptyField + } + if cfg.RequireNumericSbd && !allASCIIDigits(soBaoDanh) { + return SkipNonNumericSbd + } + return SkipNone +} + +// allASCIIDigits mirrors Rust's chars().all(|c| c.is_ascii_digit()). +// +// Deliberately not strconv.Atoi: Atoi accepts a leading sign, so "+123" would +// pass a check Rust rejects. Empty input returns true, matching Rust's all() on +// an empty iterator — the empty case is caught earlier by RequireNonemptySbd. +func allASCIIDigits(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] < '0' || s[i] > '9' { + return false + } + } + return true +} + +// ParseScores extracts subject scores from a DIEM_THI cell. +// +// Every one of the 16 patterns runs against every dataset; a subject absent from +// a given exam year never matches and stays NULL. Matching is unanchored +// first-match, like Rust's Regex::captures. +func ParseScores(diemThi string) map[string]float64 { + out := make(map[string]float64) + if diemThi == "" { + return out + } + for field, re := range schema.ScorePatterns { + m := re.FindStringSubmatch(diemThi) + if m == nil { + continue + } + v, err := strconv.ParseFloat(m[1], 64) + if err != nil { + continue + } + // Unreachable given the pattern shape, but kept for parity with + // transform.rs:136 (is_finite). + if math.IsInf(v, 0) || math.IsNaN(v) { + continue + } + out[field] = v + } + return out +} + +// ErrNoColumns is returned when the fixed-column path is used on a config that +// has no columns: mapping. Rust panics here via .expect() (transform.rs:162); +// returning an error is the Go-idiomatic equivalent and is unreachable in +// practice, since only the non-2016 path calls this. +var ErrNoColumns = errors.New("transform: config has no columns mapping") + +// TransformRow extracts one row into a ParsedRow using fixed column indices, +// the 2017-family path. 2016 uses runtime format detection instead. +func TransformRow(raw []reader.Cell, cfg *config.DatasetConfig) (*ParsedRow, error) { + cols := cfg.Columns + if cols == nil { + return nil, ErrNoColumns + } + + // Trimmed accessor, mirroring the closure at transform.rs:163-167. Out-of-range + // indices yield "" rather than an error, matching unwrap_or_default(). + get := func(idx int) string { + if idx < 0 || idx >= len(raw) { + return "" + } + return strings.TrimSpace(raw[idx].Str) + } + + hoTen := get(cols.HoTen) + ngaySinh := get(cols.NgaySinh) + soBaoDanh := get(cols.SoBaoDanh) + + // diem_thi is read WITHOUT trimming (transform.rs:172-175), unlike the three + // fields above. Harmless because the score patterns are unanchored, but it is + // the shipped behaviour — do not "tidy" it. + diemThi := "" + if cols.DiemThi >= 0 && cols.DiemThi < len(raw) { + diemThi = raw[cols.DiemThi].Str + } + + var ngaySinhOpt *string + if ngaySinh != "" { + ngaySinhOpt = &ngaySinh + } + + return &ParsedRow{ + SoBaoDanh: soBaoDanh, + HoTen: hoTen, + HoTenAscii: ToAscii(hoTen), + NgaySinh: ngaySinhOpt, + TenCumThi: nil, + GioiTinh: nil, + Scores: ParseScores(diemThi), + }, nil +} diff --git a/parser/internal/transform/transform_test.go b/parser/internal/transform/transform_test.go new file mode 100644 index 0000000..0e2e5e2 --- /dev/null +++ b/parser/internal/transform/transform_test.go @@ -0,0 +1,283 @@ +package transform + +import ( + "testing" + + "github.com/tiennm99/thptqg/parser/internal/config" + "github.com/tiennm99/thptqg/parser/internal/reader" +) + +// Ports every test in parser/src/transform.rs's test module (:201-409) — 29 in +// total, not the 20 in the :213-315 range, which covers only ToAscii. The nine +// outside that range are the ParseScores and ValidateRow cases, i.e. exactly the +// behaviours this package's traps concern. + +// --- ToAscii: the 20 cases at transform.rs:213-315 --- + +func TestToAscii(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"plain_latin", "Nguyen Van A", "nguyen van a"}, + {"nguyen_thi_hoa", "Nguyễn Thị Hoa", "nguyen thi hoa"}, + {"tran_van_duc", "Trần Văn Đức", "tran van duc"}, + {"le_thi_my_duyen", "Lê Thị Mỹ Duyên", "le thi my duyen"}, + {"pham_thi_lan", "Phạm Thị Lan", "pham thi lan"}, + {"bui_thi_thu", "Bùi Thị Thu", "bui thi thu"}, + {"hoang_van_truong", "Hoàng Văn Trường", "hoang van truong"}, + {"do_thi_ngan", "Đỗ Thị Ngân", "do thi ngan"}, + {"nguyen_van_khanh", "Nguyễn Văn Khánh", "nguyen van khanh"}, + {"trinh_thi_bich_ngoc", "Trịnh Thị Bích Ngọc", "trinh thi bich ngoc"}, + {"vu_thi_dieu", "Vũ Thị Diệu", "vu thi dieu"}, + {"nguyen_thi_tuong_vi", "Nguyễn Thị Tường Vi", "nguyen thi tuong vi"}, + {"lowercase_d_stroke", "đặng thị hằng", "dang thi hang"}, + {"uppercase_d_stroke", "ĐẶNG THỊ HẰNG", "dang thi hang"}, + {"mixed_case", "NGUYỄN VĂN AN", "nguyen van an"}, + {"tran_thi_kim_anh", "Trần Thị Kim Anh", "tran thi kim anh"}, + {"nguyen_thi_phuong_thao", "Nguyễn Thị Phương Thảo", "nguyen thi phuong thao"}, + {"le_van_long", "Lê Văn Long", "le van long"}, + {"vo_thi_xuan_mai", "Võ Thị Xuân Mai", "vo thi xuan mai"}, + {"empty_string", "", ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := ToAscii(c.in); got != c.want { + t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +// TestToAsciiUsesLiteralRangeNotUnicodeMn guards the highest-value trap in this +// package. transform.rs:56 filters the literal range U+0300..U+036F; the inline +// comment at :53 calls it "Unicode category M", but the code is the spec. +// unicode.Mn is strictly broader, so using it would strip marks Rust keeps. +// U+0654 (ARABIC HAMZA ABOVE) is in Mn but outside the range: Rust keeps it. +func TestToAsciiUsesLiteralRangeNotUnicodeMn(t *testing.T) { + const in = "aٔb" + if got := ToAscii(in); got != in { + t.Errorf("ToAscii(%q) = %q — a combining mark outside U+0300..U+036F must survive; "+ + "stripping it means unicode.Mn was used instead of the literal range", in, got) + } + // And a mark inside the range must be stripped. + if got := ToAscii("áb"); got != "ab" { + t.Errorf("ToAscii(\"a\\u0301b\") = %q, want \"ab\"", got) + } +} + +// TestToAsciiDStrokeIndependentOfNFD proves the đ/Đ replacement is a separate +// step: NFD does not decompose them, so relying on the mark filter alone loses +// the letter entirely. +func TestToAsciiDStrokeIndependentOfNFD(t *testing.T) { + for _, c := range []struct{ in, want string }{ + {"đ", "d"}, {"Đ", "d"}, {"đĐ", "dd"}, + } { + if got := ToAscii(c.in); got != c.want { + t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// --- ParseScores: transform.rs:323, :332, :342 --- + +func TestParseScoresSingle(t *testing.T) { + s := ParseScores("Toán: 8.5") + if v, ok := s["toan"]; !ok || v != 8.5 { + t.Errorf("toan = %v (present=%v), want 8.5", v, ok) + } + if _, ok := s["ngu_van"]; ok { + t.Error("ngu_van should be absent") + } +} + +func TestParseScoresMultiple(t *testing.T) { + s := ParseScores("Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9") + for field, want := range map[string]float64{"toan": 7.25, "ngu_van": 6.0, "vat_ly": 9.0} { + if v, ok := s[field]; !ok || v != want { + t.Errorf("%s = %v (present=%v), want %v", field, v, ok, want) + } + } +} + +func TestParseScoresEmptyCell(t *testing.T) { + if s := ParseScores(""); len(s) != 0 { + t.Errorf("ParseScores(\"\") = %v, want empty", s) + } +} + +// TestParseScoresRealCellShape uses the wide space runs seen in the corpus. +func TestParseScoresRealCellShape(t *testing.T) { + const cell = "Toán: 4.60 Ngữ văn: 5.50 Lịch sử: 4.50 " + s := ParseScores(cell) + if len(s) != 3 { + t.Fatalf("matched %d subjects, want 3: %v", len(s), s) + } + if s["toan"] != 4.60 || s["ngu_van"] != 5.50 || s["lich_su"] != 4.50 { + t.Errorf("got %v", s) + } +} + +// --- ValidateRow: transform.rs:359, :365, :374, :383, :393, :400 --- + +func defaultValidation() *config.ValidationCfg { + return &config.ValidationCfg{ + RequireNumericSbd: false, + RequireNonemptyName: true, + RequireNonemptySbd: true, + } +} + +func TestValidateOK(t *testing.T) { + if r := ValidateRow("Nguyen Van A", "12345678", defaultValidation(), false, false); r != SkipNone { + t.Errorf("got %v, want SkipNone", r) + } +} + +func TestValidateEmptySbd(t *testing.T) { + if r := ValidateRow("Nguyen Van A", "", defaultValidation(), false, false); r != SkipEmptyField { + t.Errorf("got %v, want SkipEmptyField", r) + } +} + +func TestValidateEmptyName(t *testing.T) { + if r := ValidateRow("", "12345678", defaultValidation(), false, false); r != SkipEmptyField { + t.Errorf("got %v, want SkipEmptyField", r) + } +} + +func TestValidateNonNumericSbdRejected(t *testing.T) { + v := defaultValidation() + v.RequireNumericSbd = true + if r := ValidateRow("Nguyen Van A", "12AB5678", v, false, false); r != SkipNonNumericSbd { + t.Errorf("got %v, want SkipNonNumericSbd", r) + } +} + +func TestValidateNumericSbdAccepted(t *testing.T) { + v := defaultValidation() + v.RequireNumericSbd = true + if r := ValidateRow("Nguyen Van A", "12345678", v, false, false); r != SkipNone { + t.Errorf("got %v, want SkipNone", r) + } +} + +func TestValidateBlankRowSkipped(t *testing.T) { + if r := ValidateRow("", "", defaultValidation(), true, true); r != SkipBlankRow { + t.Errorf("got %v, want SkipBlankRow", r) + } +} + +// TestValidateNumericSbdIsDigitScanNotAtoi: Rust uses chars().all(is_ascii_digit), +// which strconv.Atoi does not reproduce — Atoi accepts a leading sign, and would +// wrongly admit "+123". +func TestValidateNumericSbdIsDigitScanNotAtoi(t *testing.T) { + v := defaultValidation() + v.RequireNumericSbd = true + for _, sbd := range []string{"+123", "-123", "12 3", "1.0", "ABC123", "１２３"} { + if r := ValidateRow("Nguyen Van A", sbd, v, false, false); r != SkipNonNumericSbd { + t.Errorf("ValidateRow(sbd=%q) = %v, want SkipNonNumericSbd", sbd, r) + } + } +} + +// TestValidateBlankRowOnlyWhenStripEnabled: with strip_blank_rows false, an +// all-blank row falls through to the empty-field checks instead (transform.rs:109). +func TestValidateBlankRowOnlyWhenStripEnabled(t *testing.T) { + if r := ValidateRow("", "", defaultValidation(), false, true); r != SkipEmptyField { + t.Errorf("got %v, want SkipEmptyField when strip_blank_rows is off", r) + } +} + +// --- TransformRow --- + +func fixedColumnCfg() *config.DatasetConfig { + return &config.DatasetConfig{ + Columns: &config.ColumnMap{HoTen: 0, NgaySinh: 1, SoBaoDanh: 2, DiemThi: 3}, + Validation: *defaultValidation(), + } +} + +func cells(vals ...string) []reader.Cell { + out := make([]reader.Cell, len(vals)) + for i, v := range vals { + out[i] = reader.Cell{Str: v, IsEmpty: v == ""} + } + return out +} + +func TestTransformRow(t *testing.T) { + row := cells("Nguyễn Văn Đức", "04/04/1999", "51002167", "Toán: 8.5 Ngữ văn: 7") + got, err := TransformRow(row, fixedColumnCfg()) + if err != nil { + t.Fatalf("TransformRow: %v", err) + } + if got.HoTen != "Nguyễn Văn Đức" || got.HoTenAscii != "nguyen van duc" { + t.Errorf("ho_ten=%q ascii=%q", got.HoTen, got.HoTenAscii) + } + if got.SoBaoDanh != "51002167" { + t.Errorf("so_bao_danh = %q", got.SoBaoDanh) + } + if got.NgaySinh == nil || *got.NgaySinh != "04/04/1999" { + t.Errorf("ngay_sinh = %v", got.NgaySinh) + } + // 2016-only columns are never populated on the fixed-column path. + if got.TenCumThi != nil || got.GioiTinh != nil { + t.Error("ten_cum_thi and gioi_tinh must stay nil on the 2017 path") + } + if got.Scores["toan"] != 8.5 || got.Scores["ngu_van"] != 7 { + t.Errorf("scores = %v", got.Scores) + } +} + +// TestTransformRowEmptyNgaySinhBecomesNil ports transform.rs:179-183. +func TestTransformRowEmptyNgaySinhBecomesNil(t *testing.T) { + got, err := TransformRow(cells("A", "", "1", ""), fixedColumnCfg()) + if err != nil { + t.Fatalf("TransformRow: %v", err) + } + if got.NgaySinh != nil { + t.Errorf("empty ngay_sinh should be nil, got %q", *got.NgaySinh) + } +} + +// TestTransformRowShortRowYieldsEmptyFields ports the unwrap_or_default() +// behaviour at transform.rs:163-176: a row shorter than the configured indices +// yields empty strings rather than an error. +func TestTransformRowShortRowYieldsEmptyFields(t *testing.T) { + got, err := TransformRow(cells("OnlyName"), fixedColumnCfg()) + if err != nil { + t.Fatalf("TransformRow: %v", err) + } + if got.HoTen != "OnlyName" || got.SoBaoDanh != "" || got.NgaySinh != nil { + t.Errorf("got ho_ten=%q sbd=%q ngay_sinh=%v", got.HoTen, got.SoBaoDanh, got.NgaySinh) + } +} + +// TestTransformRowDiemThiIsNotTrimmed pins an asymmetry that is easy to +// "tidy away": ho_ten, ngay_sinh and so_bao_danh are trimmed through the closure +// at transform.rs:164-168, but diem_thi is read raw at :172-175. +func TestTransformRowDiemThiIsNotTrimmed(t *testing.T) { + row := cells(" A ", " 01/01/2000 ", " 123 ", " Toán: 5 ") + got, err := TransformRow(row, fixedColumnCfg()) + if err != nil { + t.Fatalf("TransformRow: %v", err) + } + if got.HoTen != "A" || got.SoBaoDanh != "123" { + t.Errorf("trimmed fields wrong: ho_ten=%q sbd=%q", got.HoTen, got.SoBaoDanh) + } + if got.NgaySinh == nil || *got.NgaySinh != "01/01/2000" { + t.Errorf("ngay_sinh = %v, want trimmed", got.NgaySinh) + } + // Untrimmed diem_thi still parses — the regexes are unanchored. + if got.Scores["toan"] != 5 { + t.Errorf("scores = %v", got.Scores) + } +} + +// TestTransformRowRequiresColumns: the fixed-column path is only reachable when +// the config has a columns: mapping. Rust panics via .expect() (transform.rs:162); +// Go returns an error instead. +func TestTransformRowRequiresColumns(t *testing.T) { + cfg := &config.DatasetConfig{Validation: *defaultValidation()} + if _, err := TransformRow(cells("A", "B", "C", "D"), cfg); err == nil { + t.Fatal("TransformRow without a columns: mapping must return an error") + } +} diff --git a/parser/internal/writer/writer.go b/parser/internal/writer/writer.go new file mode 100644 index 0000000..a820759 --- /dev/null +++ b/parser/internal/writer/writer.go @@ -0,0 +1,177 @@ +// Package writer handles SQLite output: DDL setup, INSERT OR REPLACE, VACUUM +// and the stats block — a port of parser/src/writer.rs. +// +// Every dataset writes the same canonical table (internal/schema), so there is +// exactly one insert path. Columns a dataset carries no data for bind NULL. +// +// The stats lines are reproduced verbatim because they are the operator-facing +// output of the build, and docs/deployment-guide.md points at the per-file row +// counts for troubleshooting. +package writer + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + + "github.com/tiennm99/thptqg/parser/internal/schema" + "github.com/tiennm99/thptqg/parser/internal/sqlitedb" + "github.com/tiennm99/thptqg/parser/internal/transform" +) + +// OpenDB deletes any existing database at dbPath, recreates it, and executes the +// canonical DDL. +// +// Deleting the file rather than issuing DROP TABLE mirrors build-lib.js:54 via +// writer.rs:24-30. A consequence worth knowing: a concurrent reader sees the file +// vanish mid-rebuild rather than a transactional swap. +func OpenDB(dbPath string) (*sql.DB, error) { + if _, err := os.Stat(dbPath); err == nil { + if err := os.Remove(dbPath); err != nil { + return nil, fmt.Errorf("remove existing db %s: %w", dbPath, err) + } + } + if parent := filepath.Dir(dbPath); parent != "" && parent != "." { + if err := os.MkdirAll(parent, 0o755); err != nil { + return nil, fmt.Errorf("create %s: %w", parent, err) + } + } + + db, err := sql.Open(sqlitedb.DriverName, dbPath) + if err != nil { + return nil, fmt.Errorf("open db %s: %w", dbPath, err) + } + if _, err := db.Exec(schema.DDL); err != nil { + db.Close() + return nil, fmt.Errorf("execute DDL: %w", err) + } + return db, nil +} + +// Inserter wraps a prepared INSERT statement. +// +// Rust calls conn.execute(INSERT_SQL, ...) per row (writer.rs:73), re-preparing +// each time. Preparing once is a performance choice, not a parity requirement — +// the SQL and its bindings are identical either way. +type Inserter struct{ stmt *sql.Stmt } + +// Prepare compiles the canonical INSERT against tx. +func Prepare(tx *sql.Tx) (*Inserter, error) { + stmt, err := tx.Prepare(schema.InsertSQL) + if err != nil { + return nil, fmt.Errorf("prepare insert: %w", err) + } + return &Inserter{stmt: stmt}, nil +} + +// Close releases the prepared statement. +func (i *Inserter) Close() error { return i.stmt.Close() } + +// Insert binds one parsed row and executes the INSERT. +// +// Parameter order is IdentityFields then ScoreFields. Subjects absent from +// row.Scores — and the two identity columns only the 2016 layouts populate — +// bind NULL. +func (i *Inserter) Insert(row *transform.ParsedRow) error { + args := make([]any, 0, schema.ParamCount) + args = append(args, + row.SoBaoDanh, + row.HoTen, + row.HoTenAscii, + nullableString(row.NgaySinh), + nullableString(row.TenCumThi), + nullableString(row.GioiTinh), + ) + for _, field := range schema.ScoreFields { + if v, ok := row.Scores[field]; ok { + args = append(args, v) + } else { + args = append(args, nil) + } + } + if len(args) != schema.ParamCount { + return fmt.Errorf("built %d params, want %d", len(args), schema.ParamCount) + } + if _, err := i.stmt.Exec(args...); err != nil { + return err + } + return nil +} + +func nullableString(s *string) any { + if s == nil { + return nil + } + return *s +} + +// Stats carries the counters the build loop accumulates. +type Stats struct { + SourceRows uint64 + Skipped uint64 + Errors uint64 +} + +// Finish runs VACUUM and prints the stats block. +// +// VACUUM must run AFTER the transaction commits — SQLite refuses it inside one. +// +// The wording branches on datasetLabel, which Rust derives from the input +// directory's basename (main.rs:98-101). That makes the output depend on a +// filesystem path rather than on config; it is reproduced here for parity, and +// the caller passes the label explicitly so tests are not at the mercy of a +// temp-directory name. +func Finish(db *sql.DB, dbPath string, st Stats, datasetLabel string, isOld2 bool) error { + if _, err := db.Exec("VACUUM"); err != nil { + return fmt.Errorf("vacuum: %w", err) + } + + var dbCount int64 + if err := db.QueryRow("SELECT COUNT(*) FROM student").Scan(&dbCount); err != nil { + return fmt.Errorf("count rows: %w", err) + } + + insertable := st.SourceRows - st.Skipped + + fmt.Println() + if isOld2 { + fmt.Printf("Source non-blank data rows: %d\n", st.SourceRows) + fmt.Printf(" skipped (empty/non-numeric SBD): %d\n", st.Skipped) + } else { + fmt.Printf("Source data rows (post-header): %d\n", st.SourceRows) + if containsOld(datasetLabel) { + fmt.Printf(" skipped (empty/non-numeric SBD): %d\n", st.Skipped) + } else { + fmt.Printf(" skipped (empty/invalid): %d\n", st.Skipped) + } + } + fmt.Printf(" insertable: %d\n", insertable) + fmt.Printf(" insert errors: %d\n", st.Errors) + fmt.Printf("DB rows (distinct SBD): %d\n", dbCount) + + if !containsOld(datasetLabel) && st.Errors == 0 { + gap := int64(insertable) - dbCount + if gap == 0 { + fmt.Println("Audit: OK — every source row made it in.") + } else { + fmt.Printf("Audit: %d row(s) collapsed (duplicate SBDs overwriting).\n", gap) + } + } + + var size int64 + if fi, err := os.Stat(dbPath); err == nil { + size = fi.Size() + } + fmt.Printf("Size: %.1f MB\n", float64(size)/1024.0/1024.0) + return nil +} + +func containsOld(label string) bool { + for i := 0; i+3 <= len(label); i++ { + if label[i:i+3] == "old" { + return true + } + } + return false +} diff --git a/parser/scripts/build-db.js b/parser/scripts/build-db.js deleted file mode 100644 index 9558ebb..0000000 --- a/parser/scripts/build-db.js +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env node -/** - * Build the SQLite database for one or all datasets, then gzip it. - * - * Replaces the six per-dataset npm scripts the two old projects carried. The - * dataset list comes from src/datasets.js so it is written in exactly one place. - * - * Output goes to .build/public/db/ — the directory Vite copies as its publicDir. - * Only the .gz survives: shipping a 100+ MB uncompressed database is made - * structurally impossible rather than left to a cleanup step. - * - * Usage: - * node parser/scripts/build-db.js # all four datasets - * node parser/scripts/build-db.js 2017-old # just one - */ - -import { execFileSync } from "node:child_process"; -import { mkdirSync, rmSync, existsSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { DATASET_IDS } from "../../src/datasets.js"; - -const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); -const BIN = resolve(ROOT, "parser/target/release/xlsxread"); -const OUT_DIR = resolve(ROOT, ".build/public/db"); - -const requested = process.argv.slice(2); -const unknown = requested.filter((id) => !DATASET_IDS.includes(id)); -if (unknown.length) { - console.error(`unknown dataset(s): ${unknown.join(", ")}`); - console.error(`known: ${DATASET_IDS.join(", ")}`); - process.exit(2); -} -const targets = requested.length ? requested : DATASET_IDS; - -if (!existsSync(BIN)) { - console.error(`parser binary not found at ${BIN}`); - console.error("run: npm run build:rust"); - process.exit(1); -} - -mkdirSync(OUT_DIR, { recursive: true }); - -for (const id of targets) { - const db = resolve(OUT_DIR, `${id}.db`); - - execFileSync( - BIN, - [ - "build", - "--schema", - resolve(ROOT, `parser/configs/${id}.toml`), - "--input", - resolve(ROOT, `data/${id}`), - "--output", - db, - ], - { stdio: "inherit" }, - ); - - // -9 without -k: the raw .db must not reach the published artifact. - rmSync(`${db}.gz`, { force: true }); - execFileSync("gzip", ["-9", db], { stdio: "inherit" }); - console.log(` → db/${id}.db.gz\n`); -} diff --git a/parser/scripts/check-duplicates.js b/parser/scripts/check-duplicates.js deleted file mode 100644 index c0b4a53..0000000 --- a/parser/scripts/check-duplicates.js +++ /dev/null @@ -1,30 +0,0 @@ -// One-off audit: detect content-identical Excel files via md5. -// -// BROKEN as committed — `dirs` below is a hardcoded Windows path from the -// original author's machine. Predates the repo unification; left unchanged -// rather than half-fixed. Point `dirs` at data/ to use it. -import crypto from "crypto"; -import fs from "fs"; -import path from "path"; - -const dirs = ["D:/tiennm99/thptqg2017/data"]; - -const byHash = {}; -for (const d of dirs) { - for (const f of fs.readdirSync(d)) { - const full = path.join(d, f); - if (!fs.statSync(full).isFile() || !f.endsWith(".xlsx")) continue; - const h = crypto.createHash("md5").update(fs.readFileSync(full)).digest("hex"); - (byHash[h] ||= []).push(full); - } -} - -const total = Object.values(byHash).reduce((s, a) => s + a.length, 0); -const dupes = Object.entries(byHash).filter(([, a]) => a.length > 1); -console.log(`Total files: ${total}`); -console.log(`Unique by md5: ${Object.keys(byHash).length}`); -console.log(`Duplicate groups: ${dupes.length}`); -for (const [h, a] of dupes) { - console.log(` ${h.slice(0, 12)}:`); - a.forEach((p) => console.log(` ${p}`)); -} diff --git a/parser/scripts/crawl-baotintuc.js b/parser/scripts/crawl-baotintuc.js deleted file mode 100644 index 8f4c20c..0000000 --- a/parser/scripts/crawl-baotintuc.js +++ /dev/null @@ -1,137 +0,0 @@ -// Crawl Excel score files for all 63 provinces from baotintuc.vn article: -// https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm -// Saves to data/2017/.xls (63 files). Idempotent: skips already-downloaded files. -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const OUT_DIR = path.join(__dirname, "..", "..", "data", "2017"); - -const LINKS = [ - ["An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"], - ["Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"], - ["Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"], - ["Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"], - ["Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"], - ["Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"], - ["Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"], - ["Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"], - ["Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"], - ["Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"], - ["Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"], - ["Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"], - ["Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"], - ["Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"], - ["Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"], - ["Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"], - ["Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"], - ["Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"], - ["Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"], - ["Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"], - ["Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"], - ["Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"], - ["Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"], - ["Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"], - ["Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"], - ["Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"], - ["Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"], - ["Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"], - ["Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"], - ["Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"], - ["Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"], - ["Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"], - ["Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"], - ["Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"], - ["Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"], - ["Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"], - ["Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"], - ["Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"], - ["Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"], - ["Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"], - ["Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"], - ["Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"], - ["Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"], - ["Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"], - ["Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"], - ["Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"], - ["Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"], - ["Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"], - ["Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"], - ["Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"], - ["Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"], - ["Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"], - ["Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"], - ["Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"], - ["Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"], - ["Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"], - ["Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"], - ["Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"], - ["Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"], - ["Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"], - ["Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"], - ["Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"], - ["Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"], -]; - -function slug(name) { - return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); -} - -async function downloadOne(name, url) { - const outPath = path.join(OUT_DIR, `${slug(name)}.xls`); - if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) { - return { name, url, outPath, status: "skip", size: fs.statSync(outPath).size }; - } - const res = await fetch(url, { - headers: { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", - Referer: - "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm", - }, - }); - if (!res.ok) { - return { name, url, outPath, status: "fail", httpStatus: res.status }; - } - const buf = Buffer.from(await res.arrayBuffer()); - fs.writeFileSync(outPath, buf); - return { name, url, outPath, status: "ok", size: buf.length }; -} - -async function main() { - fs.mkdirSync(OUT_DIR, { recursive: true }); - console.log(`Downloading ${LINKS.length} files to ${OUT_DIR}...`); - - const CONCURRENCY = 6; - const results = []; - let i = 0; - async function worker() { - while (i < LINKS.length) { - const idx = i++; - const [name, url] = LINKS[idx]; - try { - const r = await downloadOne(name, url); - results.push(r); - const tag = r.status === "ok" ? "✓" : r.status === "skip" ? "·" : "✗"; - const sz = r.size ? `${(r.size / 1024).toFixed(0)} KB` : ""; - console.log(` ${tag} [${idx + 1}/${LINKS.length}] ${name.padEnd(20)} ${sz} ${r.status === "fail" ? "HTTP " + r.httpStatus : ""}`); - } catch (err) { - console.log(` ✗ [${idx + 1}/${LINKS.length}] ${name}: ${err.message}`); - results.push({ name, url, status: "error", message: err.message }); - } - } - } - await Promise.all(Array.from({ length: CONCURRENCY }, worker)); - - const ok = results.filter((r) => r.status === "ok").length; - const skip = results.filter((r) => r.status === "skip").length; - const fail = results.filter((r) => r.status === "fail" || r.status === "error"); - console.log(`\nDone. ok=${ok} skip=${skip} fail=${fail.length}`); - if (fail.length) { - console.log("Failed:", fail.map((f) => f.name).join(", ")); - process.exit(1); - } -} - -main(); diff --git a/parser/scripts/db-stats.js b/parser/scripts/db-stats.js deleted file mode 100644 index 8582821..0000000 --- a/parser/scripts/db-stats.js +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env node -/** - * Dump per-dataset statistics from built SQLite databases as JSON. - * - * Used twice: once against the pre-refactor databases to capture a baseline, - * and again after the schema unification. Comparing the two outputs is what - * proves no score data was lost or invented. - * - * Schema-agnostic on purpose — columns come from PRAGMA table_info, so the same - * script runs against the old 18/20-column tables and the new 22-column one. - * - * Usage: - * node db-stats.js