diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 3b8263d..502a273 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -66,8 +66,8 @@ jobs: - name: Test assembler run: go -C assembler test ./... - # Lint covers ESLint and svelte-check; the tests cover the framework-free - # modules, including the ASCII fold that has to match the Go parser. + # The tests cover the framework-free modules, including the ASCII fold + # that has to match the Go parser. - name: Test web working-directory: web run: npm test diff --git a/CLAUDE.md b/CLAUDE.md index 01a8380..e640750 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,8 @@ Add the web checks when frontend files changed: (cd web && npm test && npm run lint && npm run build) ``` -`npm run lint` is ESLint plus `svelte-check`, so it is the type check too. +`npm run lint` is ESLint. The web app is plain JavaScript — no TypeScript, no +type-check step. **Do not run the crawler suite as part of routine verification.** The crawler is not part of the build — `data//` is committed and a crawl only refreshes it @@ -44,8 +45,8 @@ hashes every real input file. That is the point of it; do not skip it. regenerated.** A mismatch is a reader bug until proven otherwise, never a cue to refresh the file. `parser/cmd/dumpcells` narrows a failure to the cell. - **`ToAscii` filters the literal range U+0300..U+036F, not `unicode.Mn`.** It - must match `toAscii` in `web/src/lib/to-ascii.ts`, or accent-insensitive search - silently misses rows. `to-ascii.test.ts` pins the pairs both sides must agree + must match `toAscii` in `web/src/lib/to-ascii.js`, or accent-insensitive search + silently misses rows. `to-ascii.test.js` pins the pairs both sides must agree on. - **The 2016 files use four different layouts, and detection is per sheet.** Two of them publish scores in one column per subject instead of a `DIEM_THI` @@ -58,6 +59,15 @@ hashes every real input file. That is the point of it; do not skip it. used — a fallback would break the `?q=` deep links. - **`dbSizeMb` in `datasets.json` is a build guard, not just a label.** The assembler refuses to publish an artifact that falls below a ratio of it. +- **The databases ship uncompressed, as `.sqlite3`.** The browser reads + byte ranges of them, and a range of a gzip stream is not a range of the + database. The host must not apply `Content-Encoding` either — check with + `curl -sI` after a deploy. +- **Every query the site runs must be index-driven.** Over range requests an + unindexed query fetches the whole table. Hence no index on `ho_ten` (nothing + can use one), `name_word` for name search, partial indexes for the score + presets, and the footer count read from `datasets.json` instead of + `COUNT(*)`. ## Conventions diff --git a/README.md b/README.md index afcfd9f..1fcbda7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # thptqg Tra cứu điểm thi THPT Quốc gia — exam-score lookup for Vietnam's national high -school graduation exam. Client-side SQL (sql.js) over a SQLite database built +school graduation exam. Client-side SQL over a SQLite database read in place by +HTTP range request, built from the published `.xls`/`.xlsx` score files by the Go `parser` module. Where those files come from: [data pipeline](./docs/data-pipeline.md#sources). @@ -36,12 +37,12 @@ Each stage runs on its own and hands its output to the next through the stores. `datasets.json` is the contract between them. It is JSON because Go and the web app both read it and neither needs a dependency to do so; presentation stays in -`web/src/lib/datasets.ts`, keyed by id, which fails loudly if the two disagree. +`web/src/lib/datasets.js`, keyed by id, which fails loudly if the two disagree. The dataset id is one identifier end to end: ``` -data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ +data/2017/ → parser/configs/2017.yml → db/2017.sqlite3 → /thptqg/2017/ ``` ## Build @@ -85,7 +86,7 @@ Pushing to `main` runs the same steps in 2. Add `parser/configs/.yml` — sheet mode, column indices, validation guards. No SQL; the schema is canonical. 3. Add an entry to `datasets.json` with its expected row count and size -4. Add the matching presentation to `CONTENT` in `web/src/lib/datasets.ts` +4. Add the matching presentation to `CONTENT` in `web/src/lib/datasets.js` Everything else follows: the assembler, the router and the hub all read the registry, and the UI adapts to whichever columns the dataset fills. Steps 3 and 4 diff --git a/assembler/internal/databases/databases.go b/assembler/internal/databases/databases.go index 0bf077f..c294c46 100644 --- a/assembler/internal/databases/databases.go +++ b/assembler/internal/databases/databases.go @@ -1,5 +1,4 @@ -// Package databases builds, verifies and compresses one SQLite file per -// dataset. +// Package databases builds and verifies one SQLite file per dataset. // // VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra. // @@ -11,13 +10,15 @@ // // The guards below close that: a build whose row count does not match the // registry, or whose artifact is implausibly small, fails the pipeline. +// +// The databases ship uncompressed. The browser reads them a page at a time over +// HTTP range requests, and a range of a gzip stream is not a range of the +// database. package databases import ( - "compress/gzip" "database/sql" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -30,10 +31,15 @@ import ( // driverName is modernc.org/sqlite's registered name. const driverName = "sqlite" -// minSizeRatio: a gzipped database far below its usual size means a truncated -// build, even if the row count somehow passed. +// minSizeRatio: a database far below its usual size means a truncated build, +// even if the row count somehow passed. const minSizeRatio = 0.9 +// Extension is the published suffix. Not ".db": the sql.js-httpvfs ecosystem +// uses ".sqlite3", and keeping ".db" free lets the site assembly treat any +// stray .db or SQLite journal in the output as the leftover it is. +const Extension = ".sqlite3" + // Paths locates the pieces this package needs. type Paths struct { // Root is the repository root. @@ -75,7 +81,7 @@ func Build(p Paths, bin string, d registry.Dataset) error { if err := os.MkdirAll(p.OutDir, 0o755); err != nil { return err } - db := filepath.Join(p.OutDir, d.ID+".db") + db := filepath.Join(p.OutDir, d.ID+Extension) cmd := exec.Command(bin, "build", @@ -99,12 +105,11 @@ func Build(p Paths, bin string, d registry.Dataset) error { } fmt.Printf(" ✓ %s: %d rows (matches expected)\n", d.ID, rows) - gz, size, err := compress(db) + st, err := os.Stat(db) if err != nil { return fmt.Errorf("%s: %w", d.ID, err) } - - sizeMb := float64(size) / 1024 / 1024 + sizeMb := float64(st.Size()) / 1024 / 1024 if min := d.DbSizeMb * minSizeRatio; sizeMb < min { return fmt.Errorf( "%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+ @@ -112,7 +117,7 @@ func Build(p Paths, bin string, d registry.Dataset) error { d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb) } - fmt.Printf(" → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb) + fmt.Printf(" → %s (%.1f MB)\n\n", filepath.Base(db), sizeMb) return nil } @@ -131,61 +136,8 @@ func countRows(path string) (int64, error) { return n, nil } -// compress gzips path to path+".gz" and removes the original, returning the -// compressed path and its size. -// -// The source is deleted only after the compressed file is closed successfully, -// so a failure part-way through leaves the database rather than losing it. -func compress(path string) (string, int64, error) { - in, err := os.Open(path) - if err != nil { - return "", 0, err - } - defer in.Close() - - gzPath := path + ".gz" - out, err := os.Create(gzPath) - if err != nil { - return "", 0, err - } - - zw, err := gzip.NewWriterLevel(out, gzip.BestCompression) - if err != nil { - out.Close() - return "", 0, err - } - if _, err := io.Copy(zw, in); err != nil { - zw.Close() - out.Close() - os.Remove(gzPath) - return "", 0, err - } - if err := zw.Close(); err != nil { - out.Close() - os.Remove(gzPath) - return "", 0, err - } - if err := out.Close(); err != nil { - os.Remove(gzPath) - return "", 0, err - } - - if err := in.Close(); err != nil { - return "", 0, err - } - if err := os.Remove(path); err != nil { - return "", 0, fmt.Errorf("removing the uncompressed database: %w", err) - } - - st, err := os.Stat(gzPath) - if err != nil { - return "", 0, err - } - return gzPath, st.Size(), nil -} - // Clean removes staged artifacts for datasets that are no longer in the -// registry. Without this a removed dataset's .db.gz lingers in the staging +// registry. Without this a removed dataset's file lingers in the staging // directory, and the site assembly copies that directory wholesale — so the // dead database would be published again. func Clean(p Paths, keep []registry.Dataset) error { @@ -197,10 +149,9 @@ func Clean(p Paths, keep []registry.Dataset) error { return err } - wanted := make(map[string]bool, len(keep)*2) + wanted := make(map[string]bool, len(keep)) for _, d := range keep { - wanted[d.ID+".db"] = true - wanted[d.ID+".db.gz"] = true + wanted[d.ID+Extension] = true } for _, e := range entries { diff --git a/assembler/internal/databases/databases_test.go b/assembler/internal/databases/databases_test.go index 05a9c27..b14aada 100644 --- a/assembler/internal/databases/databases_test.go +++ b/assembler/internal/databases/databases_test.go @@ -1,8 +1,6 @@ package databases import ( - "compress/gzip" - "io" "os" "path/filepath" "slices" @@ -11,52 +9,12 @@ import ( "github.com/tiennm99/thptqg/assembler/internal/registry" ) -// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that -// shipping a 100+ MB uncompressed database is structurally impossible rather -// than left to a cleanup step. -func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) { - dir := t.TempDir() - src := filepath.Join(dir, "2016.db") - body := []byte("pretend this is a SQLite file") - if err := os.WriteFile(src, body, 0o644); err != nil { - t.Fatal(err) - } - - gzPath, size, err := compress(src) - if err != nil { - t.Fatal(err) - } - if gzPath != src+".gz" || size <= 0 { - t.Fatalf("gzPath=%q size=%d", gzPath, size) - } - if _, err := os.Stat(src); !os.IsNotExist(err) { - t.Error("the uncompressed database must not survive") - } - - f, err := os.Open(gzPath) - if err != nil { - t.Fatal(err) - } - defer f.Close() - zr, err := gzip.NewReader(f) - if err != nil { - t.Fatal(err) - } - got, err := io.ReadAll(zr) - if err != nil { - t.Fatal(err) - } - if string(got) != string(body) { - t.Errorf("round-trip gave %q", got) - } -} - func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) { dir := t.TempDir() for _, name := range []string{ - "2016.db.gz", "2017.db.gz", - "2017-old.db.gz", // dropped from the registry - "2017-old2.db.gz", // dropped from the registry + "2016.sqlite3", "2017.sqlite3", + "2017-old.sqlite3", // dropped from the registry + "2017-old2.sqlite3", // dropped from the registry "2016.db-journal", // interrupted run } { if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil { @@ -80,7 +38,7 @@ func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) { } slices.Sort(left) - want := []string{"2016.db.gz", "2017.db.gz"} + want := []string{"2016.sqlite3", "2017.sqlite3"} if !slices.Equal(left, want) { t.Errorf("left %v, want %v", left, want) } diff --git a/assembler/internal/site/site.go b/assembler/internal/site/site.go index 4428df2..5018b2c 100644 --- a/assembler/internal/site/site.go +++ b/assembler/internal/site/site.go @@ -20,6 +20,7 @@ import ( "regexp" "strings" + "github.com/tiennm99/thptqg/assembler/internal/databases" "github.com/tiennm99/thptqg/assembler/internal/registry" ) @@ -89,7 +90,7 @@ func Assemble(p Paths, datasets []registry.Dataset) error { if err := checkDatabasesPresent(p.Site, datasets); err != nil { return err } - if err := checkNoRawDatabases(p.Site); err != nil { + if err := checkNoStrayArtifacts(p.Site); err != nil { return err } @@ -131,10 +132,10 @@ func checkDatasetPages(siteDir string, datasets []registry.Dataset) error { func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error { var missing []string for _, d := range datasets { - gz := filepath.Join(siteDir, "db", d.ID+".db.gz") - st, err := os.Stat(gz) + file := filepath.Join(siteDir, "db", d.ID+databases.Extension) + st, err := os.Stat(file) if err != nil || st.Size() == 0 { - missing = append(missing, d.ID+".db.gz") + missing = append(missing, d.ID+databases.Extension) } } if len(missing) > 0 { @@ -146,22 +147,23 @@ func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error { return nil } -// rawDatabase matches an uncompressed SQLite artifact, including the temporary -// files SQLite leaves mid-build. -var rawDatabase = regexp.MustCompile(`\.db(-journal|-wal|-shm)?$`) +// strayArtifact matches what must never reach the output: a SQLite journal from +// an interrupted run, a database under the old .db name, or a gzipped database +// from before the switch to range requests. +var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite3)(-journal|-wal|-shm)$|\.db$|\.gz$`) -// checkNoRawDatabases rejects an uncompressed database that reached the output. +// checkNoStrayArtifacts rejects leftovers that would be published. // -// The build gzips without keeping the source, so none should exist — but the -// staging directory is copied wholesale, and a leftover from an interrupted run -// would go straight through. A raw database is 100+ MB. -func checkNoRawDatabases(siteDir string) error { +// The staging directory is copied wholesale, so anything an interrupted run left +// behind goes straight through — and each of these is 100+ MB. A gzipped +// database would also be unreadable to the site, which reads byte ranges. +func checkNoStrayArtifacts(siteDir string) error { var stray []string err := filepath.WalkDir(siteDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - if !d.IsDir() && rawDatabase.MatchString(d.Name()) { + if !d.IsDir() && strayArtifact.MatchString(d.Name()) { stray = append(stray, path) } return nil @@ -171,7 +173,7 @@ func checkNoRawDatabases(siteDir string) error { } if len(stray) > 0 { var b strings.Builder - b.WriteString("uncompressed database artefact(s) found in the site output:\n") + b.WriteString("stray database artefact(s) found in the site output:\n") for _, f := range stray { st, _ := os.Stat(f) fmt.Fprintf(&b, " %s (%.1f MB)\n", f, float64(st.Size())/1048576) diff --git a/assembler/internal/site/site_test.go b/assembler/internal/site/site_test.go index 121a49f..dae2b8e 100644 --- a/assembler/internal/site/site_test.go +++ b/assembler/internal/site/site_test.go @@ -23,7 +23,7 @@ func fakeBuild(t *testing.T, dbs ...string) Paths { } write(t, filepath.Join(dist, "_app", "immutable", "entry.js"), "console.log(1)") for _, name := range dbs { - write(t, filepath.Join(dist, "db", name), "gzipped-bytes") + write(t, filepath.Join(dist, "db", name), "sqlite-bytes") } return Paths{Web: filepath.Join(root, "web"), Dist: dist, Site: filepath.Join(root, "_site")} } @@ -39,7 +39,7 @@ func write(t *testing.T, path, body string) { } func TestAssembleProducesAPageForEveryDataset(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := Assemble(p, datasets); err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) { filepath.Join("2016", "index.html"), filepath.Join("2017", "index.html"), filepath.Join("_app", "immutable", "entry.js"), - filepath.Join("db", "2016.db.gz"), + filepath.Join("db", "2016.sqlite3"), } { if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil { t.Errorf("missing from the artifact: %s", want) @@ -61,7 +61,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) { // entry generator, which reads the same registry this does. If the two fall out // of step, that dataset's URL 404s — so the build stops instead. func TestMissingDatasetPageFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := os.RemoveAll(filepath.Join(p.Dist, "2017")); err != nil { t.Fatal(err) } @@ -80,12 +80,12 @@ func TestMissingDatasetPageFailsTheBuild(t *testing.T) { // renders, every query 404s, and CI stays green. The row-count and size guards // cannot catch this — they only run when a database was built at all. func TestMissingDatabaseFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz") // 2017 never built + p := fakeBuild(t, "2016.sqlite3") // 2017 never built err := Assemble(p, datasets) if err == nil { t.Fatal("expected an error when a database is missing") } - if !strings.Contains(err.Error(), "2017.db.gz") { + if !strings.Contains(err.Error(), "2017.sqlite3") { t.Errorf("the error should name the missing database, got: %v", err) } } @@ -93,40 +93,45 @@ func TestMissingDatabaseFailsTheBuild(t *testing.T) { // TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is // not a database. func TestEmptyDatabaseFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") - write(t, filepath.Join(p.Dist, "db", "2017.db.gz"), "") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") + write(t, filepath.Join(p.Dist, "db", "2017.sqlite3"), "") if err := Assemble(p, datasets); err == nil { t.Fatal("expected an error for a zero-byte database") } } -// TestRawDatabaseFailsTheBuild: the compression step deletes its source, so a -// raw .db here means an interrupted run left one behind — and it is 100+ MB. -func TestRawDatabaseFailsTheBuild(t *testing.T) { - for _, name := range []string{"2016.db", "2016.db-journal", "2016.db-wal", "2016.db-shm"} { +// TestStrayArtifactFailsTheBuild: a journal means an interrupted run, a .db +// means the old naming, a .gz means a database the site could not read a range +// of — and each is 100+ MB. +func TestStrayArtifactFailsTheBuild(t *testing.T) { + for _, name := range []string{ + "2016.db", "2016.sqlite3-journal", "2016.sqlite3-wal", "2016.sqlite3-shm", "2016.sqlite3.gz", + } { t.Run(name, func(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") write(t, filepath.Join(p.Dist, "db", name), "raw sqlite") err := Assemble(p, datasets) if err == nil { t.Fatalf("expected an error for %s", name) } - if !strings.Contains(err.Error(), "uncompressed") { + if !strings.Contains(err.Error(), "stray") { t.Errorf("unexpected error: %v", err) } }) } } -// TestGzipIsNotMistakenForRaw: the reject pattern is anchored, so a .db.gz must -// pass. Getting this wrong would fail every build. -func TestGzipIsNotMistakenForRaw(t *testing.T) { - if rawDatabase.MatchString("2016.db.gz") { - t.Error("a .db.gz must not be treated as an uncompressed database") +// TestPublishedDatabaseIsNotMistakenForStray: the pattern must pass the one +// file the site is built to serve. Getting this wrong would fail every build. +func TestPublishedDatabaseIsNotMistakenForStray(t *testing.T) { + if strayArtifact.MatchString("2016.sqlite3") { + t.Error("the published database must not be treated as a stray artifact") } - for _, name := range []string{"2016.db", "x.db-journal", "x.db-wal", "x.db-shm"} { - if !rawDatabase.MatchString(name) { - t.Errorf("%s should be treated as an uncompressed artifact", name) + for _, name := range []string{ + "2016.db", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm", "x.sqlite3.gz", "x.db.gz", + } { + if !strayArtifact.MatchString(name) { + t.Errorf("%s should be rejected", name) } } } @@ -142,7 +147,7 @@ func TestAssembleRejectsAMissingBuild(t *testing.T) { // TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a // previous run's leftovers cannot survive into the artifact. func TestAssembleIsIdempotent(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := Assemble(p, datasets); err != nil { t.Fatal(err) } diff --git a/assembler/internal/verify/verify.go b/assembler/internal/verify/verify.go index 3674889..ff826ec 100644 --- a/assembler/internal/verify/verify.go +++ b/assembler/internal/verify/verify.go @@ -44,8 +44,8 @@ type Result struct { // OK reports whether the two databases are logically identical. func (r Result) OK() bool { return len(r.Problems) == 0 } -// Compare checks every dataset in the registry, reading /.db.gz (or -// .db) from each side. +// Compare checks every dataset in the registry, reading /.sqlite3 +// from each side. func Compare(datasets []registry.Dataset, dirA, dirB string) ([]Result, error) { out := make([]Result, 0, len(datasets)) for _, d := range datasets { @@ -135,21 +135,26 @@ func compareOne(id, dirA, dirB string) (Result, error) { return res, nil } -// open finds .db.gz or .db in dir and returns a handle. A compressed -// database is expanded to a temporary file, since SQLite needs to seek. +// open finds .sqlite3 in dir and returns a read-only handle. +// +// A gzipped database is still expanded to a temporary file rather than +// rejected: the two sides of a comparison are often a build from before the +// switch to range requests and one from after. func open(dir, id string) (*sql.DB, func(), error) { noop := func() {} - plain := filepath.Join(dir, id+".db") - if _, err := os.Stat(plain); err == nil { - db, err := sql.Open(driverName, "file:"+plain+"?mode=ro") - return db, noop, err + for _, name := range []string{id + ".sqlite3", id + ".db"} { + plain := filepath.Join(dir, name) + if _, err := os.Stat(plain); err == nil { + db, err := sql.Open(driverName, "file:"+plain+"?mode=ro") + return db, noop, err + } } gzPath := filepath.Join(dir, id+".db.gz") f, err := os.Open(gzPath) if err != nil { - return nil, noop, fmt.Errorf("no %s.db or %s.db.gz in %s", id, id, dir) + return nil, noop, fmt.Errorf("no %s.sqlite3, %s.db or %s.db.gz in %s", id, id, id, dir) } defer f.Close() diff --git a/datasets.json b/datasets.json index ecd83e2..4543006 100644 --- a/datasets.json +++ b/datasets.json @@ -8,9 +8,10 @@ " inputs are frozen exam results, so a deviation of even one row", " means something changed unintentionally and the assembler", " refuses to publish.", - " dbSizeMb usual size of the gzipped database. Required and non-zero:", - " the assembler rejects a build that comes out far smaller, and", - " the web app shows it while the download runs.", + " dbSizeMb usual size of the published database. Required and non-zero:", + " the assembler rejects a build that comes out far smaller. The", + " file is served uncompressed and read a page at a time over", + " HTTP range requests, so nothing downloads it whole.", "", "Presentation (titles, labels, SQL presets) lives in web/src/datasets.js keyed", "by id; that file throws at load if the two lists disagree." @@ -19,12 +20,12 @@ { "id": "2016", "expectedRows": 877460, - "dbSizeMb": 45 + "dbSizeMb": 302 }, { "id": "2017", "expectedRows": 861068, - "dbSizeMb": 48 + "dbSizeMb": 247 } ] } diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 8546af6..518df28 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -181,6 +181,10 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what ## Expected row counts +The databases are written with 1 KiB pages (`PRAGMA page_size` in +`parser/internal/writer/writer.go`) because the browser fetches them a page per +HTTP request. `CHUNK_BYTES` in `web/src/lib/sqlite.svelte.js` must match. + | id | Source rows | Skipped | DB rows | | --- | --- | --- | --- | | `2016` | 877,460 | 0 | **877,460** | @@ -189,7 +193,7 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what ## Verifying a rebuild The assembler verifies itself: each database's row count must match the -figure in the table above, and each `.db.gz` must be at least 90% of its usual +figure in the table above, and each `.sqlite3` must be at least 90% of its usual size, or the build fails rather than publishing. That guard is the reason a truncated dataset cannot reach the site with a green pipeline. @@ -203,8 +207,8 @@ go -C assembler run ./cmd/assemble db # rebuild go -C assembler run ./cmd/assemble verify /tmp/before .build/public/db ``` -Each side is a directory of `.db.gz` (or `.db`); compressed databases are -expanded to a temporary file automatically. It exits non-zero on any mismatch, +Each side is a directory of `.sqlite3`; a gzipped database from before the +switch to range requests is still expanded to a temporary file automatically. It exits non-zero on any mismatch, names the first differing rows and columns, and fails rather than skipping when a dataset is absent from either side — silently comparing one of two datasets is how a gate passes without proving anything. diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 171baa4..b34a9dc 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -75,17 +75,22 @@ artifact — one missing line away from publishing it. ## Notes -- **The gzipped database is not cacheable across deploys.** Every rebuild - produces a different `.db.gz`, because SQLite does not lay pages out - deterministically. First-visit users pay the full download; later visits hit - browser cache until the next deploy. -- **GitHub Pages caps individual files at 100 MB.** The largest gzipped database - is about 48 MB. Uncompressed they run 135–234 MB and would not fit — which is - why the browser decompresses via `DecompressionStream`. -- **No server-side compression is assumed.** The app fetches the `.gz` bytes - directly rather than relying on `Content-Encoding: gzip`; Pages does not - reliably compress arbitrary paths on the fly. -- **Total artifact is about 93 MB**, well inside the 1 GB site limit. +- **The database is not cacheable across deploys.** Every rebuild lays SQLite + pages out differently, so the file changes even when the data does not. Only + the pages a query touches are fetched, so this costs far less than it used + to, but a deploy does invalidate what a returning visitor had cached. +- **The 100 MB file limit is a Git limit, not a Pages one.** It applies to + files committed to a repository; the databases are built in CI and uploaded + as a Pages artifact, and the documented Pages limits are a 1 GB published + site and 100 GB/month of bandwidth, with no per-file figure. The two + databases are 302 MB and 247 MB. +- **Total artifact is about 552 MB**, inside the 1 GB site limit but with less + headroom than before: a third dataset of this size would not fit. The fallback + is `sql.js-httpvfs`'s chunked mode, which splits a database into parts. +- **The server must not compress the databases.** Ranges of a compressed body + address the wrong bytes, and the library refuses to open a file whose HEAD + carries a `Content-Encoding`. `.sqlite3` is an unknown type to Pages, so it is + served as `application/octet-stream` and left alone — verify after a deploy. ## Rollback @@ -99,6 +104,7 @@ run rebuilds the older state. There is no data to migrate. | Blank page, 404 on assets | `paths.base` in `svelte.config.js` does not match the repo name | | `Failed to fetch database: 404` | Dataset id in `datasets.json` does not match the file in `db/` | | A route 404s | The site step did not run, or the id is missing from `datasets.json` | -| WASM fails to load | `sql.js.org` unreachable — self-host `sql-wasm.wasm` and update `SQL_WASM_URL` in `lib/sqlite.svelte.ts` | +| Database fails to open | The host compressed it. `curl -sI …/db/.sqlite3` must show no `content-encoding`; ranges of a compressed body are unusable | +| Every query is slow or huge | It is not using an index. `EXPLAIN QUERY PLAN` it: a `SCAN` means the browser is fetching the whole table | | Deploy fails on assembly | An uncompressed database artefact reached the output; the error names the files | | Missing rows after a data update | Unknown Excel header — check the per-file row counts the parser prints | diff --git a/docs/project-overview.md b/docs/project-overview.md index d2f2951..90bc5a1 100644 --- a/docs/project-overview.md +++ b/docs/project-overview.md @@ -21,10 +21,10 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the ## Constraints -- **Zero backend.** The full database (44–48 MB gzipped per dataset) is - downloaded to the browser and queried in-process. +- **Zero backend.** The database (238–289 MB per dataset) stays on the server + and the browser reads the pages a query touches over HTTP range requests. - **Read-only.** `INSERT`/`UPDATE`/`DELETE` are rejected, so nobody is misled - into thinking edits persist. `sql.js` is in-memory anyway. + into thinking edits persist. The file is fetched, never written. - **Row caps.** 100 rows for lookups, 1000 for custom SQL, to prevent browser hangs. - **Vietnamese-first UI.** App labels and data are Vietnamese; documentation is diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 544c3d6..fd6d5e3 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -1,7 +1,9 @@ # System Architecture -Static site, no backend. The browser downloads a compressed SQLite file at boot -and every query runs locally via `sql.js` (SQLite compiled to WebAssembly). +Static site, no backend. The SQLite file stays on the server and the browser +reads the pages a query touches over HTTP range requests, via `sql.js-httpvfs` +(SQLite compiled to WebAssembly behind a virtual file system). A lookup costs a +few hundred KB; nothing downloads the database. One frontend, one parser, one schema, two datasets. @@ -15,11 +17,11 @@ through. `assembler/` sequences everything from the parser onwards. data//*.xls(x) │ ▼ parser/ (Go, one binary, one config per dataset) - .build/public/db/.db - │ - ▼ assembler/ — row count must match datasets.json, then gzip - .build/public/db/.db.gz (the raw .db does not survive) + .build/public/db/.sqlite3 │ + ▼ assembler/ — row count and size must match datasets.json + .build/public/db/.sqlite3 (uncompressed: ranges of a gzip stream + │ are not ranges of the database) ▼ assembler/ → npm run build (SvelteKit static, assets = .build/public) web/dist/ │ @@ -27,7 +29,7 @@ data//*.xls(x) _site/ → GitHub Pages │ ▼ browser - sql.js (WASM) opens the .db → queries run client-side + sql.js-httpvfs asks for pages → HTTP range requests → results client-side ``` ## The dataset id @@ -35,7 +37,7 @@ data//*.xls(x) One identifier ties the whole pipeline together: ``` -data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ +data/2017/ → parser/configs/2017.yml → db/2017.sqlite3 → /thptqg/2017/ ``` `datasets.json` at the repository root declares the ids once, with the row count @@ -44,7 +46,7 @@ because the assembler is a Go program and the web app is not, and JSON is the only format both parse without a dependency. Presentation — titles, labels, search examples, SQL presets — stays in -`web/src/datasets.js`, keyed by id. That file cross-checks the two: a registry +`web/src/lib/datasets.js`, keyed by id. That file cross-checks the two: a registry entry with no content, or content for a dataset that was never built, throws at module load rather than rendering a page with no title or a link to a database that does not exist. @@ -123,13 +125,13 @@ No component contains a per-dataset conditional. Two mechanisms do the work: self-exclude wherever those languages were not sat. Anything genuinely per-dataset — title, source, database size, search examples, -SQL presets — lives in `web/src/lib/datasets.ts`. +SQL presets — lives in `web/src/lib/datasets.js`. ## Exam ID formats -`web/src/lib/query-mode.ts` decides whether a query is an exam ID or a name, and +`web/src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is shared by the dataset page and `search-form.svelte` (they previously held -separate copies and had drifted apart on exactly this rule). `query-mode.test.ts` +separate copies and had drifted apart on exactly this rule). `query-mode.test.js` covers every form in the table below. | Form | Example | Where | @@ -168,10 +170,12 @@ total descending. | Concern | Choice | Rationale | | --- | --- | --- | -| Storage | Static SQLite file | No backend; the datasets are frozen | -| Compression | gzip in CI, `DecompressionStream` in the browser | Native API, no extra library | -| WASM hosting | `sql.js.org` CDN | Smaller self-hosted artifact | -| Diacritics search | Pre-computed `ho_ten_ascii` | `LOWER(REPLACE(...))` at query time defeats the index | +| Storage | Static SQLite file, read by range request | No backend; the datasets are frozen, and a lookup needs a few pages of them | +| Compression | None | A byte range of a gzip stream is not a byte range of the database | +| WASM hosting | Bundled with the app | `sql.js-httpvfs` ships its own build; one less third-party runtime dependency | +| Diacritics search | Pre-computed `ho_ten_ascii`, indexed word by word | `LOWER(REPLACE(...))` at query time defeats the index, and `LIKE '%x%'` reads the whole table | +| Row count in the footer | Read from `datasets.json` | `COUNT(*)` scans an index — 20 MB over range requests | +| Page size | 1 KiB, matched by `requestChunkSize` | One HTTP request is one page; a row fetched by seek costs 1 KB rather than 4 KB, for about 5% more file | | SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes cannot persist; the allowlist prevents confusion | | Row caps | 100 (lookup), 1000 (SQL) | Keeps DOM render sizes reasonable | | Routing | SvelteKit file routes, prerendered | Each dataset gets a real HTML file with its own title | @@ -179,12 +183,16 @@ total descending. ## Risks and limitations -- **Database size.** 45–48 MB gzipped per dataset; slow links wait, mitigated by - a progress bar. -- **Browser memory.** The full database lives in RAM; older mobile devices may - run out. -- **`sql.js.org` dependency.** If that CDN is unreachable, the WASM fails to - load. Self-hosting `sql-wasm.wasm` and updating `SQL_WASM_URL` in - `web/src/lib/sqlite.svelte.ts` is the fix. +- **Unindexed queries are expensive.** The SQL tab can express a query that + walks the table, which over range requests means fetching 100+ MB. A byte + budget stops one before it gets that far, and the tab warns before it opens. +- **`Content-Encoding` breaks everything.** If the host ever compresses + `.sqlite3` on the wire, ranges address compressed bytes and + `sql.js-httpvfs` refuses to open the file. Verify after a deploy: + `curl -sI …/db/2016.sqlite3` must show no `content-encoding`. +- **`sql.js-httpvfs` is unmaintained** (0.8.12, September 2022) and ships its + own SQLite WASM. `sqlite-wasm-http`, on the official build, is the fallback. +- **Hosted size.** 552 MB for both datasets against the 1 GB GitHub Pages + limit; a third dataset of this size would not fit. - **Excel format drift.** A new source file with an unseen header layout needs a new branch in `parser/internal/ingest/detect2016.go` or a new config. diff --git a/parser/README.md b/parser/README.md index 23f57e1..3fea335 100644 --- a/parser/README.md +++ b/parser/README.md @@ -41,7 +41,7 @@ reader independently verifiable against a hash oracle. - `ToAscii` strips combining marks in the literal range U+0300–U+036F rather than by Unicode category. That covers every Vietnamese diacritic and must - stay identical to `toAscii` in `web/src/lib/to-ascii.ts`, or accent-insensitive + stay identical to `toAscii` in `web/src/lib/to-ascii.js`, or accent-insensitive search misses rows. - Gender is normalised to `Nam`/`Nữ`; the Cần Thơ files write `0`/`1` instead and are translated. Anything else becomes NULL. diff --git a/parser/internal/schema/schema.go b/parser/internal/schema/schema.go index 95ed9bc..5ad429d 100644 --- a/parser/internal/schema/schema.go +++ b/parser/internal/schema/schema.go @@ -18,9 +18,17 @@ import "regexp" // DDL is executed verbatim after the output database is (re)created. // -// idx_ten_cum_thi is partial, so it holds zero entries on the 2017 dataset — -// where the column is always NULL — while staying useful for the 2016 -// cluster-grouping queries. Partial indexes are SQLite-specific. +// Every index here is chosen for a database read over HTTP range requests, +// where an unindexed query downloads the table. The rules that follow from +// that: +// +// - No index on ho_ten or ho_ten_ascii. Neither substring nor prefix LIKE can +// use one (SQLite's LIKE optimisation needs a NOCASE index or +// case_sensitive_like), so both scanned the whole table. name_word replaces +// them. +// - idx_ten_cum_thi is partial, so it holds zero entries on the 2017 dataset +// — where the column is always NULL — while serving the 2016 cluster +// grouping. Partial indexes are SQLite-specific. // // This text is frozen: it decides the shape of every database the parser // produces. TestDDLIsFrozen holds an independent copy so any edit has to be @@ -50,9 +58,48 @@ CREATE TABLE student ( tieng_nhat REAL, tieng_trung REAL ); -CREATE INDEX idx_ho_ten ON student(ho_ten); -CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; + +CREATE TABLE name_word ( + word TEXT NOT NULL, + so_bao_danh TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + PRIMARY KEY (word, so_bao_danh) +) WITHOUT ROWID; + +CREATE TABLE name_word_freq ( + word TEXT PRIMARY KEY, + n INTEGER NOT NULL +) WITHOUT ROWID; +` + +// PostLoadSQL runs once the student rows are in, before VACUUM. +// +// The frequency table is what lets the site pick which word of a query to seek +// on: the vocabulary is about 4,400 words and the rarest word of a real query +// matches a few hundred rows, so seeking on it and filtering the rest inside +// name_word keeps a search to a few hundred kilobytes. +// +// The three score indexes are partial for the same reason idx_ten_cum_thi is: +// each covers only the exam year that has the column, and each costs about +// 4 MB. They exist so the SQL presets that rank by these columns seek instead +// of scanning 127 MB. +const PostLoadSQL = ` +INSERT INTO name_word_freq (word, n) + SELECT word, COUNT(*) FROM name_word GROUP BY word; +CREATE INDEX idx_toan ON student(toan) WHERE toan IS NOT NULL; +CREATE INDEX idx_khtn ON student(khtn) WHERE khtn IS NOT NULL; +CREATE INDEX idx_khxh ON student(khxh) WHERE khxh IS NOT NULL; +` + +// NameWordInsertSQL adds one row per distinct word of a candidate's ASCII name. +// +// ho_ten_ascii is carried along deliberately: a query with several words seeks +// on the rarest one and filters the others against this copy, so the whole +// match happens inside one b-tree and only the surviving rows are read from +// student. +const NameWordInsertSQL = ` +INSERT OR IGNORE INTO name_word (word, so_bao_danh, ho_ten_ascii) VALUES (?, ?, ?) ` // IdentityFields are the identity columns, in INSERT parameter order. diff --git a/parser/internal/schema/schema_test.go b/parser/internal/schema/schema_test.go index d54f72b..0cbce5c 100644 --- a/parser/internal/schema/schema_test.go +++ b/parser/internal/schema/schema_test.go @@ -109,15 +109,53 @@ CREATE TABLE student ( tieng_nhat REAL, tieng_trung REAL ); -CREATE INDEX idx_ho_ten ON student(ho_ten); -CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; + +CREATE TABLE name_word ( + word TEXT NOT NULL, + so_bao_danh TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + PRIMARY KEY (word, so_bao_danh) +) WITHOUT ROWID; + +CREATE TABLE name_word_freq ( + word TEXT PRIMARY KEY, + n INTEGER NOT NULL +) WITHOUT ROWID; ` if DDL != want { t.Errorf("DDL changed\n--- got ---\n%s\n--- want ---\n%s", DDL, want) } } +// TestNoIndexOnNameColumns: the databases are read over HTTP range requests, so +// an index that no query can use is dead weight in a file the browser pages +// through. Neither substring nor prefix LIKE can use one on these columns — +// name_word is what serves name search. +func TestNoIndexOnNameColumns(t *testing.T) { + for _, dead := range []string{"idx_ho_ten ", "idx_ho_ten_ascii"} { + if strings.Contains(DDL, dead) { + t.Errorf("DDL creates %q, which no query plan can use", dead) + } + } +} + +// TestPostLoadBuildsTheSearchTables: the frequency table is what lets a search +// pick which word to seek on, and the score indexes are what keep the SQL +// presets off a full scan. +func TestPostLoadBuildsTheSearchTables(t *testing.T) { + for _, want := range []string{ + "INSERT INTO name_word_freq", + "CREATE INDEX idx_toan", + "CREATE INDEX idx_khtn", + "CREATE INDEX idx_khxh", + } { + if !strings.Contains(PostLoadSQL, want) { + t.Errorf("PostLoadSQL is missing %q", want) + } + } +} + // TestScorePatternsMatchScores exercises each pattern against the shape the // DIEM_THI cell actually carries, including the wide runs of spaces seen in the // real corpus. diff --git a/parser/internal/writer/writer.go b/parser/internal/writer/writer.go index 339918e..eb8ea86 100644 --- a/parser/internal/writer/writer.go +++ b/parser/internal/writer/writer.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/tiennm99/thptqg/parser/internal/schema" "github.com/tiennm99/thptqg/parser/internal/sqlitedb" @@ -42,6 +43,19 @@ func OpenDB(dbPath string) (*sql.DB, error) { if err != nil { return nil, fmt.Errorf("open db %s: %w", dbPath, err) } + // Before the DDL, because a page size cannot change once a table exists — + // only the VACUUM in Finish could rewrite it, and only to this same value. + // + // 1 KiB rather than SQLite's 4 KiB default because the browser reads this + // file a page at a time over HTTP: a row fetched by index seek costs one + // page, so a search that returns 100 scattered rows transfers 100 KB + // instead of 400 KB. It costs about 5% file size, and both sql.js-httpvfs + // and sqlite-wasm-http recommend it. web/src/lib/sqlite.svelte.ts must + // request the same size. + if _, err := db.Exec("PRAGMA page_size = 1024"); err != nil { + db.Close() + return nil, fmt.Errorf("set page size: %w", err) + } if _, err := db.Exec(schema.DDL); err != nil { db.Close() return nil, fmt.Errorf("execute DDL: %w", err) @@ -110,10 +124,77 @@ type Stats struct { Errors uint64 } -// Finish runs VACUUM and prints the stats block. +// BuildNameIndex fills name_word from the student rows, one entry per distinct +// word of each ASCII name. // -// VACUUM must run AFTER the transaction commits — SQLite refuses it inside one. +// A second pass rather than a write alongside each insert: a repeated exam +// number replaces its earlier row, and the words of the row it replaced would +// otherwise stay behind pointing at a name that is no longer there. +func BuildNameIndex(db *sql.DB) error { + rows, err := db.Query("SELECT so_bao_danh, ho_ten_ascii FROM student") + if err != nil { + return fmt.Errorf("read names: %w", err) + } + defer rows.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin name index: %w", err) + } + stmt, err := tx.Prepare(schema.NameWordInsertSQL) + if err != nil { + tx.Rollback() + return fmt.Errorf("prepare name index: %w", err) + } + + var words uint64 + seen := make(map[string]struct{}, 8) + for rows.Next() { + var sbd, ascii string + if err := rows.Scan(&sbd, &ascii); err != nil { + tx.Rollback() + return fmt.Errorf("scan name: %w", err) + } + clear(seen) + for _, w := range strings.Fields(ascii) { + if _, dup := seen[w]; dup { + continue + } + seen[w] = struct{}{} + if _, err := stmt.Exec(w, sbd, ascii); err != nil { + tx.Rollback() + return fmt.Errorf("insert name word: %w", err) + } + words++ + } + } + if err := rows.Err(); err != nil { + tx.Rollback() + return fmt.Errorf("read names: %w", err) + } + if err := stmt.Close(); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit name index: %w", err) + } + + if _, err := db.Exec(schema.PostLoadSQL); err != nil { + return fmt.Errorf("post-load statements: %w", err) + } + fmt.Printf("Name index: %d words\n", words) + return nil +} + +// Finish builds the derived tables, runs VACUUM and prints the stats block. +// +// VACUUM must run AFTER the transaction commits — SQLite refuses it inside one +// — and after the name index, so the file is laid out in one pass. func Finish(db *sql.DB, dbPath string, st Stats) error { + if err := BuildNameIndex(db); err != nil { + return err + } if _, err := db.Exec("VACUUM"); err != nil { return fmt.Errorf("vacuum: %w", err) } diff --git a/plans/260814-1200-httpvfs-range-queries/plan.md b/plans/260814-1200-httpvfs-range-queries/plan.md new file mode 100644 index 0000000..2ba55e2 --- /dev/null +++ b/plans/260814-1200-httpvfs-range-queries/plan.md @@ -0,0 +1,75 @@ +# Serve the databases over HTTP range requests + +Status: implemented, unverified in a browser + +The whole-database download is gone. `sql.js-httpvfs` reads the pages a query +touches, so the databases ship raw as `.sqlite3` — a byte range of a gzip +stream is not a byte range of a database. + +## Why the schema had to change first + +Measured on the real 2016 database (223.5 MB before, 4 KB pages, 27 rows/page): + +| Query | Plan before | Would have fetched | +| --- | --- | --- | +| `so_bao_danh = ?` | SEARCH via PK | ~20 KB | +| `ho_ten_ascii LIKE '%x%'` | SCAN | 127 MB | +| `ho_ten_ascii LIKE 'x%'` | SCAN — the LIKE optimisation needs a NOCASE index | 127 MB | +| `COUNT(*)` | covering scan of idx_ho_ten_ascii | 20 MB | +| preset `ORDER BY toan DESC LIMIT 10` | SCAN + temp b-tree | 127 MB | + +So substring search was impossible, prefix search was no better, and the footer +count alone cost 20 MB per page load. + +## What shipped + +**Parser.** `name_word(word, so_bao_danh, ho_ten_ascii)` WITHOUT ROWID — the +table is the index — plus `name_word_freq(word, n)` and partial indexes on +`toan`, `khtn`, `khxh`. Dropped `idx_ho_ten` and `idx_ho_ten_ascii`: no query +plan could use either. + +877,460 names hold 2.87M word entries over a vocabulary of 4,397. A search asks +the frequency table which word is rarest, seeks on that one, and filters the +rest against the `ho_ten_ascii` copy inside the same b-tree — so "buu loc" still +finds "Nguyễn Bửu Lộc", in a few hundred KB. + +| Segment | 2016 | +| --- | --- | +| `student` | 137.5 MB | +| `name_word` | 98.7 MB | +| `idx_ten_cum_thi` | 38.1 MB | +| PK autoindex | 15.3 MB | +| `idx_toan` | 12.6 MB | +| **total** | **302.4 MB** (2017: 247.3 MB) | + +Written with 1 KiB pages, so a row reached by an index seek costs one 1 KB +request instead of 4 KB: 6.3 rows share a page rather than 27, which is what +turns a 100-row search from ~400 KB of row fetches into ~100 KB. + +**Assembler.** Publishes uncompressed; the size guard reads the raw size; the +stray-artifact check now rejects journals, `.db` and `.gz`. + +**Web.** `RemoteDatabase` wraps `createDbWorker`. The search tab runs with a +25 MB byte budget, the SQL tab asks for consent and then gets 250 MB, and the +bytes fetched are shown next to the query time. The footer count comes from +`datasets.json`. + +## Verified + +- Row counts unchanged: 877,460 and 861,068, both through the assembler guards. +- Every query the app issues is index-driven, checked with `EXPLAIN QUERY PLAN`: + `SEARCH w USING PRIMARY KEY (word>? AND word.sqlite3` — would eat every chunk**, `site.go:135-138,153`, `datasets.ts:91-97`, `datasets.json`, ~15 tests | +| Library swap | `sqlite.svelte.ts:1-3,50-58,76,82`, `package.json:15`; all consumers go through `RemoteDatabase`, so the blast radius is one file | + +## Options evaluated + +### A. page_size 1024 + requestChunkSize 1024 — ADOPTED + +- Upstream consensus: phiresky and mmomtchev both recommend 1024. +- Honest sizing for *our* pattern: row fetches 400 KB → 100 KB per search; + the index walk is sequential so bytes are unchanged and only the request + count rises, which prefetch read-heads collapse. Net ≈ 300 KB saved per + search — bandwidth, not latency. +- Cost: ~10 lines; file size +5% (528 → ~555 MB total, still under the 1 GB + Pages limit); both databases rebuilt. + +### B. serverMode chunked — REJECTED + +- Only benefit is CDN cache efficiency. GitHub Pages serves everything with + `Cache-Control: max-age=600`, and each deploy relays out SQLite pages anyway, + so cross-deploy caching is zero either way. +- Cost: split step, config JSON, `Clean()`/guards/`dbOf()`/`datasets.json` + rework, ~15 tests. +- Complexity buying a benefit the host cancels. Revisit only behind a CDN with + long TTLs. + +### C. swap to sqlite-wasm-http — DEFERRED + +- For: maintained (Dec 2025), official SQLite WASM instead of a 2022 fork; + matches this repo's posture on stale dependencies. Swap is one file. +- Against: its differentiator (shared cache) needs COOP/COEP headers GitHub + Pages cannot send, so we would get the synchronous fallback and the + maintenance benefit only. And the current integration has never run in a + browser — swapping now means two unverified variables and no way to tell + which broke. +- Revisit after the current build is verified live. + +## Decision + +Adopt **A only**. + +## Implementation notes + +1. `PRAGMA page_size = 1024` in `writer.OpenDB`, between `sql.Open` and + `db.Exec(schema.DDL)`. The existing VACUUM in `Finish` applies it. +2. `CHUNK_BYTES = 1024` in `web/src/lib/sqlite.svelte.ts`; it feeds + `requestChunkSize` and must equal the page size. +3. Rebuild both databases, update `dbSizeMb` in `datasets.json` to the new + sizes, re-run the assembler guards. + +## Risks + +- The benefit is arithmetic plus upstream authority, not measurement. The byte + counter in the SQL tab is the check, once deployed. +- Prefetch deliberately overfetches ahead of the cursor, so the 25 MB search + budget may trip earlier than a strict page count suggests. Tune after a real + measurement, not before. + +## Unresolved questions + +1. Does 1024 actually beat 4096 for our queries in a browser? +2. Is Fastly's caching of ranges over a 300 MB object good enough that chunked + mode stays unnecessary? diff --git a/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md b/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md new file mode 100644 index 0000000..ef11e7c --- /dev/null +++ b/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md @@ -0,0 +1,165 @@ +# Research Report: sql.js-httpvfs best practices + +Conducted 2026-08-14 13:17 (Asia/Saigon). Context: two static SQLite files +(2016 = 288.6 MB, 2017 = 237.7 MB) on GitHub Pages, branch +`feat/httpvfs-range-queries`. + +## Executive summary + +Our implementation matches upstream guidance on the thing that matters most — +index design — and diverges on one measurable parameter: **page size**. Both +phiresky (sql.js-httpvfs) and mmomtchev (sqlite-wasm-http) recommend +`page_size = 1024`; we shipped 4096. For our access pattern (~100 scattered +single-row reads per search) that is a real 4× overfetch on the row-fetch half +of a query. + +Two findings reduce risk rather than add work. Prefetching with three virtual +read heads makes a sequential scan cost a *logarithmic* number of requests, so +our byte estimates hold but latency is better than assumed. And the canonical +demo hosts a **670 MiB** database on GitHub Pages, so 289 MB is precedented. + +One finding is new and worth a decision: **chunked mode** (split file + JSON +config) exists specifically to make CDN caching effective for large databases, +which matters because GitHub Pages serves everything with `Cache-Control: +max-age=600`. + +## Methodology + +- Sources: 5 (1 primary blog, 2 project READMEs, 1 recent practitioner + writeup, 1 search on Pages/Fastly caching), plus direct reading of the + installed `sql.js-httpvfs@0.8.12` bundle in a previous session. +- Date range: 2021 (canonical post) → March 2026 (practitioner writeup). +- Gemini CLI absent → WebSearch/WebFetch. + +## Key findings + +### 1. Page size: recommended 1024, we use 4096 + +Both projects say the same thing. phiresky set 1 KiB pages "to balance request +overhead against bandwidth efficiency"; sqlite-wasm-http says "it is highly +recommended to decrease your SQLite page size to 1024 bytes for maximum +performance" (`PRAGMA page_size=1024; VACUUM`). + +`requestChunkSize` must match the page size. + +Measured on our 2016 file *before* the schema change: + +| page_size | file size | +| --- | --- | +| 1024 | 235.8 MB | +| 4096 | 223.5 MB | +| 8192 | 221.8 MB | + +So 1024 costs ~5.5% file size. What it buys: a scattered row read fetches 1 KB +instead of 4 KB. Our search does ~100 of those, so the row-fetch half of a +search drops from ~400 KB to ~100 KB. The index-walk half is sequential and +benefits from prefetch either way. + +**Verdict: switch to 1024 + `requestChunkSize: 1024`.** Our workload is +dominated by scattered single-row reads, which is exactly the case small pages +serve. + +### 2. Prefetch changes request count, not bytes + +"Three separate virtual read heads" detect sequential access and grow request +sizes exponentially, so "index scans or table scans reading more than a few KiB +of data will only cause a number of requests that is logarithmic in the total +byte length." + +Consequence for our analysis: a full table scan still transfers ~127 MB (bytes +are bytes), but in tens of requests rather than tens of thousands. Our byte +budget is the right guardrail; a request-count budget would not be. + +### 3. Index design — we already comply + +- Covering indexes: put every column the query needs *in* the index, else + SQLite does "another random access (unpredictable) read and thus HTTP request + to retrieve the actual value for every data point". This is exactly why + `name_word` carries `ho_ten_ascii`. +- Column order decides which lookups are cheap. +- Verify with `EXPLAIN QUERY PLAN`; a `SCAN` means the whole table crosses the + network. We did this for every query the app issues. + +### 4. Chunked mode exists for CDN caching + +`serverMode: "chunked"` splits the database into parts (10 MB is the commonly +cited size) with a JSON config. Stated benefit: "CDN caching much more +effective" for large databases. + +Relevant because **GitHub Pages sets `Cache-Control: max-age=600`** — ten +minutes — on everything. Range responses are cached by Fastly per object; with +one 289 MB object the practical caching story is weaker than with 29 chunks +that a CDN edge can hold whole. Chunked mode also sidesteps any future per-file +concern. + +Cost: a build step to split files + emit config, and every deploy invalidates +all chunks anyway (SQLite page layout is not deterministic across rebuilds). + +### 5. Hosting facts confirmed + +- Range requests work on Pages "out of the box" — matches our own probe (206 + + correct `Content-Range`). +- CORS headers (`Access-Control-Allow-Origin`, `Access-Control-Allow-Headers: + Range`) only matter cross-origin. Ours is same-origin — non-issue. +- 670 MiB database on Pages is the canonical demo. 289 MB is not exotic. +- `Content-Encoding` remains the one fatal case: the library discards + `Content-Length` and throws when a HEAD carries a non-identity encoding. + Unknown extensions like `.sqlite3` are served `application/octet-stream` and + left alone. + +### 6. Alternatives + +| | sql.js-httpvfs | sqlite-wasm-http | +| --- | --- | --- | +| WASM base | own sql.js fork (~3.36 era) | official `@sqlite.org/sqlite-wasm` | +| Last release | 0.8.12, Sept 2022 | 1.2.0, Dec 2023; activity into Dec 2025 | +| Self-description | "demo-level code… not for high stability" | "experimental" | +| Concurrency | one worker | multiple connections, shared cache | +| Shared cache needs | — | `SharedArrayBuffer` → COOP/COEP headers | +| On GitHub Pages | works | works, but **Pages cannot set COOP/COEP**, so it falls back to the synchronous backend without shared cache | +| Module format | CJS+ESM | **ES6 only** | + +Both are self-declared experimental. sqlite-wasm-http's advantage (maintained, +official WASM) is real; its headline feature (shared cache) is unavailable on +Pages precisely because Pages cannot send cross-origin isolation headers. + +## Implementation recommendations + +1. **Change page size to 1024 and `requestChunkSize` to 1024.** Parser sets + `PRAGMA page_size=1024` before DDL; VACUUM already runs. Cost ~+5% file + size; benefit ~4× less overfetch per row read. +2. **Keep `serverMode: "full"` for now.** Chunked mode's benefit is CDN cache + efficiency, which Pages' 10-minute TTL blunts. Revisit if measured repeat- + visit cost is bad. +3. **Keep sql.js-httpvfs.** Switching to sqlite-wasm-http buys a maintained + dependency but loses nothing we use, and its differentiator does not work on + Pages. Note it as the escape hatch. +4. **Verify `Content-Encoding` after deploy** — the single fatal hosting case. +5. Keep the byte budget; drop any idea of a request-count budget. + +## Common pitfalls + +- Unindexed query → whole table over the network. `EXPLAIN QUERY PLAN` is the + check. +- Page size mismatched with `requestChunkSize` → every logical page read spans + two requests. +- Serving the database compressed → library refuses to open it. +- Assuming CDN caching helps: on Pages, `max-age=600`. + +## References + +- [Hosting SQLite databases on GitHub Pages — phiresky](https://phiresky.github.io/blog/2021/hosting-sqlite-databases-on-github-pages/) +- [phiresky/sql.js-httpvfs](https://github.com/phiresky/sql.js-httpvfs) +- [mmomtchev/sqlite-wasm-http](https://github.com/mmomtchev/sqlite-wasm-http) +- [Query SQLite on GitHub Pages with sql.js-httpvfs (Mar 2026)](https://recca0120.github.io/en/2026/03/07/sql-js-httpvfs-static-hosting/) +- [sqlite3 WebAssembly documentation](https://sqlite.org/wasm) +- [GitHub Pages asset caching discussion](https://github.com/orgs/community/discussions/11884) + +## Unresolved questions + +1. Does 1024 measurably beat 4096 *for our queries*? Only a browser with the + byte counter can answer; the estimate says yes for row fetches. +2. Does Fastly cache 206 responses for a 289 MB object well enough that chunked + mode is unnecessary? Needs a deployed measurement. +3. Does the read-head prefetch overfetch on our index-range walks (fetching + ahead beyond `LIMIT 100`)? Unknown without instrumentation. diff --git a/web/eslint.config.js b/web/eslint.config.js index 2175062..6378a41 100644 --- a/web/eslint.config.js +++ b/web/eslint.config.js @@ -1,12 +1,10 @@ import js from "@eslint/js"; import svelte from "eslint-plugin-svelte"; import globals from "globals"; -import ts from "typescript-eslint"; import svelteConfig from "./svelte.config.js"; -export default ts.config( +export default [ js.configs.recommended, - ...ts.configs.recommended, ...svelte.configs.recommended, { languageOptions: { @@ -15,16 +13,11 @@ export default ts.config( }, { // Svelte files and rune modules are parsed by svelte-eslint-parser, which - // needs the TypeScript parser handed to it for `lang="ts"` blocks. - files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"], + // needs the project's svelte.config.js to resolve aliases and runes. + files: ["**/*.svelte", "**/*.svelte.js"], languageOptions: { - parserOptions: { - parser: ts.parser, - projectService: true, - extraFileExtensions: [".svelte"], - svelteConfig, - }, + parserOptions: { svelteConfig }, }, }, { ignores: ["dist/", ".svelte-kit/", "node_modules/"] }, -); +]; diff --git a/web/tsconfig.json b/web/jsconfig.json similarity index 63% rename from web/tsconfig.json rename to web/jsconfig.json index 4344710..21df54b 100644 --- a/web/tsconfig.json +++ b/web/jsconfig.json @@ -1,14 +1,11 @@ { "extends": "./.svelte-kit/tsconfig.json", "compilerOptions": { - "allowJs": true, - "checkJs": true, + "moduleResolution": "bundler", + "resolveJsonModule": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, "skipLibCheck": true, - "sourceMap": true, - "strict": true, - "moduleResolution": "bundler" + "sourceMap": true } } diff --git a/web/package-lock.json b/web/package-lock.json index 40eb2ce..6f55121 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,7 +8,7 @@ "name": "thptqg-web", "version": "1.0.0", "dependencies": { - "sql.js": "^1.14.1" + "sql.js-httpvfs": "^0.8.12" }, "devDependencies": { "@eslint/js": "^9.39.4", @@ -16,15 +16,11 @@ "@sveltejs/kit": "^2.70.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.3.3", - "@types/sql.js": "^1.4.9", "eslint": "^9.39.4", "eslint-plugin-svelte": "^3.14.0", "globals": "^17.4.0", "svelte": "^5.56.9", - "svelte-check": "^4.4.3", "tailwindcss": "^4.3.3", - "typescript": "^5.9.3", - "typescript-eslint": "^8.48.2", "vite": "^7.1.14", "vitest": "^4.1.10" } @@ -1187,16 +1183,6 @@ } } }, - "node_modules/@sveltejs/load-config": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", - "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 18.0.0" - } - }, "node_modules/@sveltejs/vite-plugin-svelte": { "version": "6.2.4", "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.4.tgz", @@ -1818,13 +1804,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/emscripten": { - "version": "1.41.5", - "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", - "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.9", "dev": true, @@ -1835,27 +1814,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" - } - }, - "node_modules/@types/sql.js": { - "version": "1.4.11", - "resolved": "https://registry.npmjs.org/@types/sql.js/-/sql.js-1.4.11.tgz", - "integrity": "sha512-QXIx38p2ZThJaK9vP5ZdqdlRe1FG9I8SmCZOS7FHfB/2qPAjZwkL7/vlfPg6N/oWHuuOaGg/P/IRwfP2W0kWVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/emscripten": "*", - "@types/node": "*" - } - }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -1863,301 +1821,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.67.0.tgz", - "integrity": "sha512-Un7Heoyj65NREbKAyIrFxeM143NZpExWmy1Nep4DLeQOeLlTeumPjoNKnBrU5D5moWXbPJgRa5Uwcdu0faVNGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/type-utils": "8.67.0", - "@typescript-eslint/utils": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "ignore": "^7.0.5", - "natural-compare": "^1.4.0", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^8.67.0", - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", - "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.67.0.tgz", - "integrity": "sha512-fUBfTuuEulWqX6V8+O3PtScV01tzYYRUDTAirHFKoRAt7nOzoGiPt0M/bB47wWNy0coOOcgEwAMUtBpykMxl6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/project-service": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.67.0.tgz", - "integrity": "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.67.0", - "@typescript-eslint/types": "^8.67.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.67.0.tgz", - "integrity": "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.67.0.tgz", - "integrity": "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.67.0.tgz", - "integrity": "sha512-aVWDXbRmdXO9siTfX4ditQI1T9+zVcNazT48EJCD0v40/9RIFoUgZ05CmGEq9H2gixRpjUn/iplwvlcvutJW/Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/utils": "8.67.0", - "debug": "^4.4.3", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/types": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.67.0.tgz", - "integrity": "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.67.0.tgz", - "integrity": "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/project-service": "8.67.0", - "@typescript-eslint/tsconfig-utils": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "debug": "^4.4.3", - "minimatch": "^10.2.2", - "semver": "^7.7.3", - "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.8" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@typescript-eslint/utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.67.0.tgz", - "integrity": "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.67.0.tgz", - "integrity": "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "eslint-visitor-keys": "^5.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/@vitest/expect": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", @@ -2401,22 +2064,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -2443,6 +2090,12 @@ "dev": true, "license": "MIT" }, + "node_modules/comlink": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.2.tgz", + "integrity": "sha512-OxGdvBmJuNKSCMO4NTl1L47VRp6xn2wG4F/2hYzB6tiCb709otOxtEYCSvK80PtjODfXXZu8ds+Nw5kVCjqd2g==", + "license": "Apache-2.0" + }, "node_modules/concat-map": { "version": "0.0.1", "dev": true, @@ -3183,16 +2836,6 @@ "node": "*" } }, - "node_modules/mri": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", - "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -3489,20 +3132,6 @@ "node": ">=6" } }, - "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "dev": true, @@ -3557,19 +3186,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/sade": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", - "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mri": "^1.1.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/set-cookie-parser": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.2.tgz", @@ -3626,9 +3242,14 @@ "node": ">=0.10.0" } }, - "node_modules/sql.js": { - "version": "1.14.1", - "license": "MIT" + "node_modules/sql.js-httpvfs": { + "version": "0.8.12", + "resolved": "https://registry.npmjs.org/sql.js-httpvfs/-/sql.js-httpvfs-0.8.12.tgz", + "integrity": "sha512-lcEBc2q0psFRfdCx8Di22oUIkkv5MUIaVO/fGCj/Jjx6YQDKVylQEcjd7NSSbmINHTRwVkm/vWP8uuevT7Rkkw==", + "license": "Apache-2.0", + "dependencies": { + "comlink": "^4.3.0" + } }, "node_modules/stackback": { "version": "0.0.2", @@ -3694,31 +3315,6 @@ "node": ">=18" } }, - "node_modules/svelte-check": { - "version": "4.7.6", - "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", - "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "@sveltejs/load-config": "^0.2.3", - "chokidar": "^4.0.1", - "fdir": "^6.2.0", - "picocolors": "^1.0.0", - "sade": "^1.7.4" - }, - "bin": { - "svelte-check": "bin/svelte-check" - }, - "engines": { - "node": ">= 18.0.0" - }, - "peerDependencies": { - "svelte": "^4.0.0 || ^5.0.0-next.0", - "typescript": "^5.0.0 || ^6.0.0" - } - }, "node_modules/svelte-eslint-parser": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-1.8.0.tgz", @@ -3836,19 +3432,6 @@ "node": ">=6" } }, - "node_modules/ts-api-utils": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", - "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.12" - }, - "peerDependencies": { - "typescript": ">=4.8.4" - } - }, "node_modules/type-check": { "version": "0.4.0", "dev": true, @@ -3860,51 +3443,6 @@ "node": ">= 0.8.0" } }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/typescript-eslint": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.67.0.tgz", - "integrity": "sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/eslint-plugin": "8.67.0", - "@typescript-eslint/parser": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/utils": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", - "dev": true, - "license": "MIT" - }, "node_modules/uri-js": { "version": "4.4.1", "dev": true, diff --git a/web/package.json b/web/package.json index a3f8851..646acc1 100644 --- a/web/package.json +++ b/web/package.json @@ -9,10 +9,10 @@ "build": "vite build", "preview": "vite preview", "test": "vitest run", - "lint": "eslint . && svelte-check --tsconfig ./tsconfig.json" + "lint": "eslint ." }, "dependencies": { - "sql.js": "^1.14.1" + "sql.js-httpvfs": "^0.8.12" }, "devDependencies": { "@eslint/js": "^9.39.4", @@ -20,15 +20,11 @@ "@sveltejs/kit": "^2.70.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.3.3", - "@types/sql.js": "^1.4.9", "eslint": "^9.39.4", "eslint-plugin-svelte": "^3.14.0", "globals": "^17.4.0", "svelte": "^5.56.9", - "svelte-check": "^4.4.3", "tailwindcss": "^4.3.3", - "typescript": "^5.9.3", - "typescript-eslint": "^8.48.2", "vite": "^7.1.14", "vitest": "^4.1.10" } diff --git a/web/src/lib/admission-blocks.ts b/web/src/lib/admission-blocks.js similarity index 88% rename from web/src/lib/admission-blocks.ts rename to web/src/lib/admission-blocks.js index 98f5142..b99c600 100644 --- a/web/src/lib/admission-blocks.ts +++ b/web/src/lib/admission-blocks.js @@ -1,15 +1,3 @@ -import type { Student, SubjectKey } from "./types"; - -export type AdmissionBlock = { code: string; subjects: SubjectKey[]; label: string }; -export type ComputedBlock = { - code: string; - label: string; - total: number; - parts: { key: SubjectKey; score: number }[]; -}; -export type TierKey = "common" | "uncommon" | "rare" | "epic" | "legendary" | "prismatic"; -export type Tier = { key: TierKey; symbol: string; label: string }; - // Vietnamese university admission-block ("khối thi") definitions, per Circular // 03/2017/TT-BGDĐT. Each block is the sum of exactly 3 subject scores. // @@ -17,7 +5,7 @@ export type Tier = { key: TierKey; symbol: string; label: string }; // skips any block with a missing subject score, so blocks needing GDCD // self-exclude on 2016 rows, and blocks needing a language the candidate did // not sit self-exclude row by row. -export const ADMISSION_BLOCKS: AdmissionBlock[] = [ +export const ADMISSION_BLOCKS = [ { code: "A00", subjects: ["toan", "vat_ly", "hoa_hoc"], label: "Toán + Lý + Hóa" }, { code: "A01", subjects: ["toan", "vat_ly", "tieng_anh"], label: "Toán + Lý + Anh" }, { code: "A02", subjects: ["toan", "vat_ly", "sinh_hoc"], label: "Toán + Lý + Sinh" }, @@ -76,14 +64,13 @@ export const ADMISSION_BLOCKS: AdmissionBlock[] = [ // Returns { code, label, total, parts:[{key,score}] } for every block where // the student has scores for all three subjects, sorted by total desc. -export function computeBlocks(student: Student): ComputedBlock[] { - const out: ComputedBlock[] = []; +export function computeBlocks(student) { + const out = []; for (const b of ADMISSION_BLOCKS) { const parts = b.subjects.map((k) => ({ key: k, score: student[k] })); if (parts.some((p) => p.score === null || p.score === undefined)) continue; - const whole = parts as { key: SubjectKey; score: number }[]; - const total = whole.reduce((s, p) => s + p.score, 0); - out.push({ code: b.code, label: b.label, total, parts: whole }); + const total = parts.reduce((s, p) => s + p.score, 0); + out.push({ code: b.code, label: b.label, total, parts }); } return out.sort((a, b) => b.total - a.total); } @@ -93,7 +80,7 @@ export function computeBlocks(student: Student): ComputedBlock[] { // whatever the other scores are, so that boundary is inclusive while the rest // are exclusive upper bounds. Every tier carries a unicode symbol as well as a // color, so the meaning is never conveyed by color alone. -export function scoreTier(score: number | null | undefined): Tier | null { +export function scoreTier(score) { if (score === null || score === undefined) return null; if (score <= 1) return { key: "common", symbol: "·", label: "Điểm liệt" }; if (score < 5) return { key: "uncommon", symbol: "○", label: "Chưa đạt" }; diff --git a/web/src/lib/admission-blocks.test.ts b/web/src/lib/admission-blocks.test.js similarity index 96% rename from web/src/lib/admission-blocks.test.ts rename to web/src/lib/admission-blocks.test.js index d78f189..8239e6e 100644 --- a/web/src/lib/admission-blocks.test.ts +++ b/web/src/lib/admission-blocks.test.js @@ -1,9 +1,8 @@ import { describe, expect, it } from "vitest"; import { ADMISSION_BLOCKS, computeBlocks, scoreTier } from "./admission-blocks"; -import type { Student } from "./types"; /** A row with every column NULL, so a test only states the scores it needs. */ -function student(scores: Partial): Student { +function student(scores) { return { so_bao_danh: "49008235", ho_ten: "Nguyễn Văn A", diff --git a/web/src/lib/components/custom-query.svelte b/web/src/lib/components/custom-query.svelte index be44eee..dd0734c 100644 --- a/web/src/lib/components/custom-query.svelte +++ b/web/src/lib/components/custom-query.svelte @@ -1,29 +1,26 @@ -
@@ -97,7 +93,7 @@ type="button" class="btn-chip rounded-md" onclick={() => runPreset(preset.sql)} - {disabled} + disabled={disabled || running} > {preset.label} @@ -111,7 +107,7 @@ class="query-form mb-4" onsubmit={(e) => { e.preventDefault(); - execute(sql); + void execute(sql); }} > -
- {#if execTime !== null} {rows.length} kết quả · {execTime}ms {/if} + {#if db?.lastCost} + + + Truy vấn này: {db.lastCost.requests} yêu cầu · {formatBytes(db.lastCost.bytes)} + · Phiên: {db.requests} yêu cầu · {formatBytes(db.bytesRead)} + + {/if}
@@ -145,7 +148,7 @@ [&>th]:bg-surface-alt [&>th]:px-2 [&>th]:py-2.5 [&>th]:text-left [&>th]:whitespace-nowrap" > - {#each columns as col, i (i)} + {#each columns as col (col)} {col} {/each} @@ -155,9 +158,9 @@ - {#each row as cell, ci (ci)} + {#each columns as col (col)} - {cell === null ? "NULL" : String(cell)} + {row[col] === null ? "NULL" : String(row[col])} {/each} diff --git a/web/src/lib/components/score-table.svelte b/web/src/lib/components/score-table.svelte index b1b2139..16bbf71 100644 --- a/web/src/lib/components/score-table.svelte +++ b/web/src/lib/components/score-table.svelte @@ -1,11 +1,10 @@ - diff --git a/web/src/routes/[dataset]/+page.ts b/web/src/routes/[dataset]/+page.js similarity index 71% rename from web/src/routes/[dataset]/+page.ts rename to web/src/routes/[dataset]/+page.js index 055bd77..b9d5a57 100644 --- a/web/src/routes/[dataset]/+page.ts +++ b/web/src/routes/[dataset]/+page.js @@ -1,15 +1,14 @@ import { error } from "@sveltejs/kit"; import { DATASETS, datasetById } from "$lib/datasets"; -import type { EntryGenerator, PageLoad } from "./$types"; /** * One prerendered page per registered dataset. The registry is the repository * root's datasets.json, so a dataset the assembler builds is a page that exists, * and one it does not is a 404 at build time rather than a blank page. */ -export const entries: EntryGenerator = () => DATASETS.map((d) => ({ dataset: d.id })); +export const entries = () => DATASETS.map((d) => ({ dataset: d.id })); -export const load: PageLoad = ({ params }) => { +export const load = ({ params }) => { const dataset = datasetById(params.dataset); if (!dataset) throw error(404, `Unknown dataset: ${params.dataset}`); return { dataset }; diff --git a/web/src/routes/[dataset]/+page.svelte b/web/src/routes/[dataset]/+page.svelte index ce0720b..64d5b71 100644 --- a/web/src/routes/[dataset]/+page.svelte +++ b/web/src/routes/[dataset]/+page.svelte @@ -1,4 +1,4 @@ -