From dbc23c25c52e2230b0595c23c0bf19038fd7120a Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 14 Aug 2026 12:42:48 +0700 Subject: [PATCH 1/5] feat: read the databases over HTTP range requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser downloaded 45 MB of gzipped SQLite before it could answer anything. Now sql.js-httpvfs asks for the pages a query touches and the databases ship uncompressed as .sqlite3 — a byte range of a gzip stream is not a byte range of a database. That only works if every query the site issues is index-driven, and measured against the real 2016 file, most were not: so_bao_danh = ? SEARCH via PK ~20 KB ho_ten_ascii LIKE '%x%' SCAN 127 MB ho_ten_ascii LIKE 'x%' SCAN 127 MB COUNT(*) covering index scan 20 MB ORDER BY toan DESC LIMIT 10 SCAN + temp b-tree 127 MB Prefix LIKE scans because SQLite's LIKE optimisation needs a NOCASE index; a range comparison does use the index. So the schema changed to suit the access pattern rather than the search changing to suit the schema. name_word holds one row per word of each name, WITHOUT ROWID so the table is the index, carrying ho_ten_ascii so a multi-word query is resolved inside a single b-tree. name_word_freq says which word of a query is rarest — the vocabulary is 4,397 words across 2.87M entries, so "buu loc" seeks on 287 entries rather than walking the 300,000 that "thi" would. Searching by any word of a name survives, at a few hundred KB a query. idx_ho_ten and idx_ho_ten_ascii are gone: no plan could use either. Partial indexes on toan, khtn and khxh cost 12 MB and keep the SQL presets off a full scan. The footer's candidate count now comes from datasets.json instead of COUNT(*). 2016 grows 223.5 MB to 288.6 MB, 2017 162.7 MB to 237.7 MB, and the site is 528 MB against the 1 GB GitHub Pages limit. Row counts are unchanged. The SQL tab is the one place a user can still write a query that reads the whole table, so it asks before it opens, runs under a byte budget that stops a runaway query, and shows what each query actually fetched. Verified: row counts through the assembler guards, every app query index-driven under EXPLAIN QUERY PLAN, and GitHub Pages returning 206 with a correct Content-Range. Not verified in a browser — this machine has none — and the library refuses to open a file the host compresses, so the deployed response headers need a look. --- CLAUDE.md | 9 + README.md | 5 +- assembler/internal/databases/databases.go | 87 ++------ .../internal/databases/databases_test.go | 50 +---- assembler/internal/site/site.go | 30 +-- assembler/internal/site/site_test.go | 51 ++--- assembler/internal/verify/verify.go | 23 ++- datasets.json | 11 +- docs/data-pipeline.md | 6 +- docs/deployment-guide.md | 30 +-- docs/project-overview.md | 6 +- docs/system-architecture.md | 47 +++-- parser/internal/schema/schema.go | 57 ++++- parser/internal/schema/schema_test.go | 42 +++- parser/internal/writer/writer.go | 72 ++++++- .../260814-1200-httpvfs-range-queries/plan.md | 71 +++++++ web/package-lock.json | 21 +- web/package.json | 4 +- web/src/lib/components/custom-query.svelte | 72 ++++--- web/src/lib/datasets.ts | 12 +- web/src/lib/search.ts | 93 +++++++++ web/src/lib/sqlite.svelte.ts | 152 ++++++++------ web/src/lib/types.ts | 5 + web/src/routes/[dataset]/+page.svelte | 195 ++++++++++-------- 24 files changed, 743 insertions(+), 408 deletions(-) create mode 100644 plans/260814-1200-httpvfs-range-queries/plan.md create mode 100644 web/src/lib/search.ts diff --git a/CLAUDE.md b/CLAUDE.md index 01a8380..a6b5962 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,6 +58,15 @@ hashes every real input file. That is the point of it; do not skip it. used — a fallback would break the `?q=` deep links. - **`dbSizeMb` in `datasets.json` is a build guard, not just a label.** The assembler refuses to publish an artifact that falls below a ratio of it. +- **The databases ship uncompressed, as `.sqlite3`.** The browser reads + byte ranges of them, and a range of a gzip stream is not a range of the + database. The host must not apply `Content-Encoding` either — check with + `curl -sI` after a deploy. +- **Every query the site runs must be index-driven.** Over range requests an + unindexed query fetches the whole table. Hence no index on `ho_ten` (nothing + can use one), `name_word` for name search, partial indexes for the score + presets, and the footer count read from `datasets.json` instead of + `COUNT(*)`. ## Conventions diff --git a/README.md b/README.md index afcfd9f..f578951 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # thptqg Tra cứu điểm thi THPT Quốc gia — exam-score lookup for Vietnam's national high -school graduation exam. Client-side SQL (sql.js) over a SQLite database built +school graduation exam. Client-side SQL over a SQLite database read in place by +HTTP range request, built from the published `.xls`/`.xlsx` score files by the Go `parser` module. Where those files come from: [data pipeline](./docs/data-pipeline.md#sources). @@ -41,7 +42,7 @@ app both read it and neither needs a dependency to do so; presentation stays in The dataset id is one identifier end to end: ``` -data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ +data/2017/ → parser/configs/2017.yml → db/2017.sqlite3 → /thptqg/2017/ ``` ## Build diff --git a/assembler/internal/databases/databases.go b/assembler/internal/databases/databases.go index 0bf077f..c294c46 100644 --- a/assembler/internal/databases/databases.go +++ b/assembler/internal/databases/databases.go @@ -1,5 +1,4 @@ -// Package databases builds, verifies and compresses one SQLite file per -// dataset. +// Package databases builds and verifies one SQLite file per dataset. // // VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra. // @@ -11,13 +10,15 @@ // // The guards below close that: a build whose row count does not match the // registry, or whose artifact is implausibly small, fails the pipeline. +// +// The databases ship uncompressed. The browser reads them a page at a time over +// HTTP range requests, and a range of a gzip stream is not a range of the +// database. package databases import ( - "compress/gzip" "database/sql" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -30,10 +31,15 @@ import ( // driverName is modernc.org/sqlite's registered name. const driverName = "sqlite" -// minSizeRatio: a gzipped database far below its usual size means a truncated -// build, even if the row count somehow passed. +// minSizeRatio: a database far below its usual size means a truncated build, +// even if the row count somehow passed. const minSizeRatio = 0.9 +// Extension is the published suffix. Not ".db": the sql.js-httpvfs ecosystem +// uses ".sqlite3", and keeping ".db" free lets the site assembly treat any +// stray .db or SQLite journal in the output as the leftover it is. +const Extension = ".sqlite3" + // Paths locates the pieces this package needs. type Paths struct { // Root is the repository root. @@ -75,7 +81,7 @@ func Build(p Paths, bin string, d registry.Dataset) error { if err := os.MkdirAll(p.OutDir, 0o755); err != nil { return err } - db := filepath.Join(p.OutDir, d.ID+".db") + db := filepath.Join(p.OutDir, d.ID+Extension) cmd := exec.Command(bin, "build", @@ -99,12 +105,11 @@ func Build(p Paths, bin string, d registry.Dataset) error { } fmt.Printf(" ✓ %s: %d rows (matches expected)\n", d.ID, rows) - gz, size, err := compress(db) + st, err := os.Stat(db) if err != nil { return fmt.Errorf("%s: %w", d.ID, err) } - - sizeMb := float64(size) / 1024 / 1024 + sizeMb := float64(st.Size()) / 1024 / 1024 if min := d.DbSizeMb * minSizeRatio; sizeMb < min { return fmt.Errorf( "%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+ @@ -112,7 +117,7 @@ func Build(p Paths, bin string, d registry.Dataset) error { d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb) } - fmt.Printf(" → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb) + fmt.Printf(" → %s (%.1f MB)\n\n", filepath.Base(db), sizeMb) return nil } @@ -131,61 +136,8 @@ func countRows(path string) (int64, error) { return n, nil } -// compress gzips path to path+".gz" and removes the original, returning the -// compressed path and its size. -// -// The source is deleted only after the compressed file is closed successfully, -// so a failure part-way through leaves the database rather than losing it. -func compress(path string) (string, int64, error) { - in, err := os.Open(path) - if err != nil { - return "", 0, err - } - defer in.Close() - - gzPath := path + ".gz" - out, err := os.Create(gzPath) - if err != nil { - return "", 0, err - } - - zw, err := gzip.NewWriterLevel(out, gzip.BestCompression) - if err != nil { - out.Close() - return "", 0, err - } - if _, err := io.Copy(zw, in); err != nil { - zw.Close() - out.Close() - os.Remove(gzPath) - return "", 0, err - } - if err := zw.Close(); err != nil { - out.Close() - os.Remove(gzPath) - return "", 0, err - } - if err := out.Close(); err != nil { - os.Remove(gzPath) - return "", 0, err - } - - if err := in.Close(); err != nil { - return "", 0, err - } - if err := os.Remove(path); err != nil { - return "", 0, fmt.Errorf("removing the uncompressed database: %w", err) - } - - st, err := os.Stat(gzPath) - if err != nil { - return "", 0, err - } - return gzPath, st.Size(), nil -} - // Clean removes staged artifacts for datasets that are no longer in the -// registry. Without this a removed dataset's .db.gz lingers in the staging +// registry. Without this a removed dataset's file lingers in the staging // directory, and the site assembly copies that directory wholesale — so the // dead database would be published again. func Clean(p Paths, keep []registry.Dataset) error { @@ -197,10 +149,9 @@ func Clean(p Paths, keep []registry.Dataset) error { return err } - wanted := make(map[string]bool, len(keep)*2) + wanted := make(map[string]bool, len(keep)) for _, d := range keep { - wanted[d.ID+".db"] = true - wanted[d.ID+".db.gz"] = true + wanted[d.ID+Extension] = true } for _, e := range entries { diff --git a/assembler/internal/databases/databases_test.go b/assembler/internal/databases/databases_test.go index 05a9c27..b14aada 100644 --- a/assembler/internal/databases/databases_test.go +++ b/assembler/internal/databases/databases_test.go @@ -1,8 +1,6 @@ package databases import ( - "compress/gzip" - "io" "os" "path/filepath" "slices" @@ -11,52 +9,12 @@ import ( "github.com/tiennm99/thptqg/assembler/internal/registry" ) -// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that -// shipping a 100+ MB uncompressed database is structurally impossible rather -// than left to a cleanup step. -func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) { - dir := t.TempDir() - src := filepath.Join(dir, "2016.db") - body := []byte("pretend this is a SQLite file") - if err := os.WriteFile(src, body, 0o644); err != nil { - t.Fatal(err) - } - - gzPath, size, err := compress(src) - if err != nil { - t.Fatal(err) - } - if gzPath != src+".gz" || size <= 0 { - t.Fatalf("gzPath=%q size=%d", gzPath, size) - } - if _, err := os.Stat(src); !os.IsNotExist(err) { - t.Error("the uncompressed database must not survive") - } - - f, err := os.Open(gzPath) - if err != nil { - t.Fatal(err) - } - defer f.Close() - zr, err := gzip.NewReader(f) - if err != nil { - t.Fatal(err) - } - got, err := io.ReadAll(zr) - if err != nil { - t.Fatal(err) - } - if string(got) != string(body) { - t.Errorf("round-trip gave %q", got) - } -} - func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) { dir := t.TempDir() for _, name := range []string{ - "2016.db.gz", "2017.db.gz", - "2017-old.db.gz", // dropped from the registry - "2017-old2.db.gz", // dropped from the registry + "2016.sqlite3", "2017.sqlite3", + "2017-old.sqlite3", // dropped from the registry + "2017-old2.sqlite3", // dropped from the registry "2016.db-journal", // interrupted run } { if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil { @@ -80,7 +38,7 @@ func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) { } slices.Sort(left) - want := []string{"2016.db.gz", "2017.db.gz"} + want := []string{"2016.sqlite3", "2017.sqlite3"} if !slices.Equal(left, want) { t.Errorf("left %v, want %v", left, want) } diff --git a/assembler/internal/site/site.go b/assembler/internal/site/site.go index 4428df2..5018b2c 100644 --- a/assembler/internal/site/site.go +++ b/assembler/internal/site/site.go @@ -20,6 +20,7 @@ import ( "regexp" "strings" + "github.com/tiennm99/thptqg/assembler/internal/databases" "github.com/tiennm99/thptqg/assembler/internal/registry" ) @@ -89,7 +90,7 @@ func Assemble(p Paths, datasets []registry.Dataset) error { if err := checkDatabasesPresent(p.Site, datasets); err != nil { return err } - if err := checkNoRawDatabases(p.Site); err != nil { + if err := checkNoStrayArtifacts(p.Site); err != nil { return err } @@ -131,10 +132,10 @@ func checkDatasetPages(siteDir string, datasets []registry.Dataset) error { func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error { var missing []string for _, d := range datasets { - gz := filepath.Join(siteDir, "db", d.ID+".db.gz") - st, err := os.Stat(gz) + file := filepath.Join(siteDir, "db", d.ID+databases.Extension) + st, err := os.Stat(file) if err != nil || st.Size() == 0 { - missing = append(missing, d.ID+".db.gz") + missing = append(missing, d.ID+databases.Extension) } } if len(missing) > 0 { @@ -146,22 +147,23 @@ func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error { return nil } -// rawDatabase matches an uncompressed SQLite artifact, including the temporary -// files SQLite leaves mid-build. -var rawDatabase = regexp.MustCompile(`\.db(-journal|-wal|-shm)?$`) +// strayArtifact matches what must never reach the output: a SQLite journal from +// an interrupted run, a database under the old .db name, or a gzipped database +// from before the switch to range requests. +var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite3)(-journal|-wal|-shm)$|\.db$|\.gz$`) -// checkNoRawDatabases rejects an uncompressed database that reached the output. +// checkNoStrayArtifacts rejects leftovers that would be published. // -// The build gzips without keeping the source, so none should exist — but the -// staging directory is copied wholesale, and a leftover from an interrupted run -// would go straight through. A raw database is 100+ MB. -func checkNoRawDatabases(siteDir string) error { +// The staging directory is copied wholesale, so anything an interrupted run left +// behind goes straight through — and each of these is 100+ MB. A gzipped +// database would also be unreadable to the site, which reads byte ranges. +func checkNoStrayArtifacts(siteDir string) error { var stray []string err := filepath.WalkDir(siteDir, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - if !d.IsDir() && rawDatabase.MatchString(d.Name()) { + if !d.IsDir() && strayArtifact.MatchString(d.Name()) { stray = append(stray, path) } return nil @@ -171,7 +173,7 @@ func checkNoRawDatabases(siteDir string) error { } if len(stray) > 0 { var b strings.Builder - b.WriteString("uncompressed database artefact(s) found in the site output:\n") + b.WriteString("stray database artefact(s) found in the site output:\n") for _, f := range stray { st, _ := os.Stat(f) fmt.Fprintf(&b, " %s (%.1f MB)\n", f, float64(st.Size())/1048576) diff --git a/assembler/internal/site/site_test.go b/assembler/internal/site/site_test.go index 121a49f..dae2b8e 100644 --- a/assembler/internal/site/site_test.go +++ b/assembler/internal/site/site_test.go @@ -23,7 +23,7 @@ func fakeBuild(t *testing.T, dbs ...string) Paths { } write(t, filepath.Join(dist, "_app", "immutable", "entry.js"), "console.log(1)") for _, name := range dbs { - write(t, filepath.Join(dist, "db", name), "gzipped-bytes") + write(t, filepath.Join(dist, "db", name), "sqlite-bytes") } return Paths{Web: filepath.Join(root, "web"), Dist: dist, Site: filepath.Join(root, "_site")} } @@ -39,7 +39,7 @@ func write(t *testing.T, path, body string) { } func TestAssembleProducesAPageForEveryDataset(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := Assemble(p, datasets); err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) { filepath.Join("2016", "index.html"), filepath.Join("2017", "index.html"), filepath.Join("_app", "immutable", "entry.js"), - filepath.Join("db", "2016.db.gz"), + filepath.Join("db", "2016.sqlite3"), } { if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil { t.Errorf("missing from the artifact: %s", want) @@ -61,7 +61,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) { // entry generator, which reads the same registry this does. If the two fall out // of step, that dataset's URL 404s — so the build stops instead. func TestMissingDatasetPageFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := os.RemoveAll(filepath.Join(p.Dist, "2017")); err != nil { t.Fatal(err) } @@ -80,12 +80,12 @@ func TestMissingDatasetPageFailsTheBuild(t *testing.T) { // renders, every query 404s, and CI stays green. The row-count and size guards // cannot catch this — they only run when a database was built at all. func TestMissingDatabaseFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz") // 2017 never built + p := fakeBuild(t, "2016.sqlite3") // 2017 never built err := Assemble(p, datasets) if err == nil { t.Fatal("expected an error when a database is missing") } - if !strings.Contains(err.Error(), "2017.db.gz") { + if !strings.Contains(err.Error(), "2017.sqlite3") { t.Errorf("the error should name the missing database, got: %v", err) } } @@ -93,40 +93,45 @@ func TestMissingDatabaseFailsTheBuild(t *testing.T) { // TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is // not a database. func TestEmptyDatabaseFailsTheBuild(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") - write(t, filepath.Join(p.Dist, "db", "2017.db.gz"), "") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") + write(t, filepath.Join(p.Dist, "db", "2017.sqlite3"), "") if err := Assemble(p, datasets); err == nil { t.Fatal("expected an error for a zero-byte database") } } -// TestRawDatabaseFailsTheBuild: the compression step deletes its source, so a -// raw .db here means an interrupted run left one behind — and it is 100+ MB. -func TestRawDatabaseFailsTheBuild(t *testing.T) { - for _, name := range []string{"2016.db", "2016.db-journal", "2016.db-wal", "2016.db-shm"} { +// TestStrayArtifactFailsTheBuild: a journal means an interrupted run, a .db +// means the old naming, a .gz means a database the site could not read a range +// of — and each is 100+ MB. +func TestStrayArtifactFailsTheBuild(t *testing.T) { + for _, name := range []string{ + "2016.db", "2016.sqlite3-journal", "2016.sqlite3-wal", "2016.sqlite3-shm", "2016.sqlite3.gz", + } { t.Run(name, func(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") write(t, filepath.Join(p.Dist, "db", name), "raw sqlite") err := Assemble(p, datasets) if err == nil { t.Fatalf("expected an error for %s", name) } - if !strings.Contains(err.Error(), "uncompressed") { + if !strings.Contains(err.Error(), "stray") { t.Errorf("unexpected error: %v", err) } }) } } -// TestGzipIsNotMistakenForRaw: the reject pattern is anchored, so a .db.gz must -// pass. Getting this wrong would fail every build. -func TestGzipIsNotMistakenForRaw(t *testing.T) { - if rawDatabase.MatchString("2016.db.gz") { - t.Error("a .db.gz must not be treated as an uncompressed database") +// TestPublishedDatabaseIsNotMistakenForStray: the pattern must pass the one +// file the site is built to serve. Getting this wrong would fail every build. +func TestPublishedDatabaseIsNotMistakenForStray(t *testing.T) { + if strayArtifact.MatchString("2016.sqlite3") { + t.Error("the published database must not be treated as a stray artifact") } - for _, name := range []string{"2016.db", "x.db-journal", "x.db-wal", "x.db-shm"} { - if !rawDatabase.MatchString(name) { - t.Errorf("%s should be treated as an uncompressed artifact", name) + for _, name := range []string{ + "2016.db", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm", "x.sqlite3.gz", "x.db.gz", + } { + if !strayArtifact.MatchString(name) { + t.Errorf("%s should be rejected", name) } } } @@ -142,7 +147,7 @@ func TestAssembleRejectsAMissingBuild(t *testing.T) { // TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a // previous run's leftovers cannot survive into the artifact. func TestAssembleIsIdempotent(t *testing.T) { - p := fakeBuild(t, "2016.db.gz", "2017.db.gz") + p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3") if err := Assemble(p, datasets); err != nil { t.Fatal(err) } diff --git a/assembler/internal/verify/verify.go b/assembler/internal/verify/verify.go index 3674889..ff826ec 100644 --- a/assembler/internal/verify/verify.go +++ b/assembler/internal/verify/verify.go @@ -44,8 +44,8 @@ type Result struct { // OK reports whether the two databases are logically identical. func (r Result) OK() bool { return len(r.Problems) == 0 } -// Compare checks every dataset in the registry, reading /.db.gz (or -// .db) from each side. +// Compare checks every dataset in the registry, reading /.sqlite3 +// from each side. func Compare(datasets []registry.Dataset, dirA, dirB string) ([]Result, error) { out := make([]Result, 0, len(datasets)) for _, d := range datasets { @@ -135,21 +135,26 @@ func compareOne(id, dirA, dirB string) (Result, error) { return res, nil } -// open finds .db.gz or .db in dir and returns a handle. A compressed -// database is expanded to a temporary file, since SQLite needs to seek. +// open finds .sqlite3 in dir and returns a read-only handle. +// +// A gzipped database is still expanded to a temporary file rather than +// rejected: the two sides of a comparison are often a build from before the +// switch to range requests and one from after. func open(dir, id string) (*sql.DB, func(), error) { noop := func() {} - plain := filepath.Join(dir, id+".db") - if _, err := os.Stat(plain); err == nil { - db, err := sql.Open(driverName, "file:"+plain+"?mode=ro") - return db, noop, err + for _, name := range []string{id + ".sqlite3", id + ".db"} { + plain := filepath.Join(dir, name) + if _, err := os.Stat(plain); err == nil { + db, err := sql.Open(driverName, "file:"+plain+"?mode=ro") + return db, noop, err + } } gzPath := filepath.Join(dir, id+".db.gz") f, err := os.Open(gzPath) if err != nil { - return nil, noop, fmt.Errorf("no %s.db or %s.db.gz in %s", id, id, dir) + return nil, noop, fmt.Errorf("no %s.sqlite3, %s.db or %s.db.gz in %s", id, id, id, dir) } defer f.Close() diff --git a/datasets.json b/datasets.json index ecd83e2..fef7161 100644 --- a/datasets.json +++ b/datasets.json @@ -8,9 +8,10 @@ " inputs are frozen exam results, so a deviation of even one row", " means something changed unintentionally and the assembler", " refuses to publish.", - " dbSizeMb usual size of the gzipped database. Required and non-zero:", - " the assembler rejects a build that comes out far smaller, and", - " the web app shows it while the download runs.", + " dbSizeMb usual size of the published database. Required and non-zero:", + " the assembler rejects a build that comes out far smaller. The", + " file is served uncompressed and read a page at a time over", + " HTTP range requests, so nothing downloads it whole.", "", "Presentation (titles, labels, SQL presets) lives in web/src/datasets.js keyed", "by id; that file throws at load if the two lists disagree." @@ -19,12 +20,12 @@ { "id": "2016", "expectedRows": 877460, - "dbSizeMb": 45 + "dbSizeMb": 289 }, { "id": "2017", "expectedRows": 861068, - "dbSizeMb": 48 + "dbSizeMb": 238 } ] } diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 8546af6..d199eda 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -189,7 +189,7 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what ## Verifying a rebuild The assembler verifies itself: each database's row count must match the -figure in the table above, and each `.db.gz` must be at least 90% of its usual +figure in the table above, and each `.sqlite3` must be at least 90% of its usual size, or the build fails rather than publishing. That guard is the reason a truncated dataset cannot reach the site with a green pipeline. @@ -203,8 +203,8 @@ go -C assembler run ./cmd/assemble db # rebuild go -C assembler run ./cmd/assemble verify /tmp/before .build/public/db ``` -Each side is a directory of `.db.gz` (or `.db`); compressed databases are -expanded to a temporary file automatically. It exits non-zero on any mismatch, +Each side is a directory of `.sqlite3`; a gzipped database from before the +switch to range requests is still expanded to a temporary file automatically. It exits non-zero on any mismatch, names the first differing rows and columns, and fails rather than skipping when a dataset is absent from either side — silently comparing one of two datasets is how a gate passes without proving anything. diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 171baa4..0cf3049 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -75,17 +75,22 @@ artifact — one missing line away from publishing it. ## Notes -- **The gzipped database is not cacheable across deploys.** Every rebuild - produces a different `.db.gz`, because SQLite does not lay pages out - deterministically. First-visit users pay the full download; later visits hit - browser cache until the next deploy. -- **GitHub Pages caps individual files at 100 MB.** The largest gzipped database - is about 48 MB. Uncompressed they run 135–234 MB and would not fit — which is - why the browser decompresses via `DecompressionStream`. -- **No server-side compression is assumed.** The app fetches the `.gz` bytes - directly rather than relying on `Content-Encoding: gzip`; Pages does not - reliably compress arbitrary paths on the fly. -- **Total artifact is about 93 MB**, well inside the 1 GB site limit. +- **The database is not cacheable across deploys.** Every rebuild lays SQLite + pages out differently, so the file changes even when the data does not. Only + the pages a query touches are fetched, so this costs far less than it used + to, but a deploy does invalidate what a returning visitor had cached. +- **The 100 MB file limit is a Git limit, not a Pages one.** It applies to + files committed to a repository; the databases are built in CI and uploaded + as a Pages artifact, and the documented Pages limits are a 1 GB published + site and 100 GB/month of bandwidth, with no per-file figure. The two + databases are 289 MB and 238 MB. +- **Total artifact is about 528 MB**, inside the 1 GB site limit but with less + headroom than before: a third dataset of this size would not fit. The fallback + is `sql.js-httpvfs`'s chunked mode, which splits a database into parts. +- **The server must not compress the databases.** Ranges of a compressed body + address the wrong bytes, and the library refuses to open a file whose HEAD + carries a `Content-Encoding`. `.sqlite3` is an unknown type to Pages, so it is + served as `application/octet-stream` and left alone — verify after a deploy. ## Rollback @@ -99,6 +104,7 @@ run rebuilds the older state. There is no data to migrate. | Blank page, 404 on assets | `paths.base` in `svelte.config.js` does not match the repo name | | `Failed to fetch database: 404` | Dataset id in `datasets.json` does not match the file in `db/` | | A route 404s | The site step did not run, or the id is missing from `datasets.json` | -| WASM fails to load | `sql.js.org` unreachable — self-host `sql-wasm.wasm` and update `SQL_WASM_URL` in `lib/sqlite.svelte.ts` | +| Database fails to open | The host compressed it. `curl -sI …/db/.sqlite3` must show no `content-encoding`; ranges of a compressed body are unusable | +| Every query is slow or huge | It is not using an index. `EXPLAIN QUERY PLAN` it: a `SCAN` means the browser is fetching the whole table | | Deploy fails on assembly | An uncompressed database artefact reached the output; the error names the files | | Missing rows after a data update | Unknown Excel header — check the per-file row counts the parser prints | diff --git a/docs/project-overview.md b/docs/project-overview.md index d2f2951..90bc5a1 100644 --- a/docs/project-overview.md +++ b/docs/project-overview.md @@ -21,10 +21,10 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the ## Constraints -- **Zero backend.** The full database (44–48 MB gzipped per dataset) is - downloaded to the browser and queried in-process. +- **Zero backend.** The database (238–289 MB per dataset) stays on the server + and the browser reads the pages a query touches over HTTP range requests. - **Read-only.** `INSERT`/`UPDATE`/`DELETE` are rejected, so nobody is misled - into thinking edits persist. `sql.js` is in-memory anyway. + into thinking edits persist. The file is fetched, never written. - **Row caps.** 100 rows for lookups, 1000 for custom SQL, to prevent browser hangs. - **Vietnamese-first UI.** App labels and data are Vietnamese; documentation is diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 544c3d6..d57b38d 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -1,7 +1,9 @@ # System Architecture -Static site, no backend. The browser downloads a compressed SQLite file at boot -and every query runs locally via `sql.js` (SQLite compiled to WebAssembly). +Static site, no backend. The SQLite file stays on the server and the browser +reads the pages a query touches over HTTP range requests, via `sql.js-httpvfs` +(SQLite compiled to WebAssembly behind a virtual file system). A lookup costs a +few hundred KB; nothing downloads the database. One frontend, one parser, one schema, two datasets. @@ -15,11 +17,11 @@ through. `assembler/` sequences everything from the parser onwards. data//*.xls(x) │ ▼ parser/ (Go, one binary, one config per dataset) - .build/public/db/.db - │ - ▼ assembler/ — row count must match datasets.json, then gzip - .build/public/db/.db.gz (the raw .db does not survive) + .build/public/db/.sqlite3 │ + ▼ assembler/ — row count and size must match datasets.json + .build/public/db/.sqlite3 (uncompressed: ranges of a gzip stream + │ are not ranges of the database) ▼ assembler/ → npm run build (SvelteKit static, assets = .build/public) web/dist/ │ @@ -27,7 +29,7 @@ data//*.xls(x) _site/ → GitHub Pages │ ▼ browser - sql.js (WASM) opens the .db → queries run client-side + sql.js-httpvfs asks for pages → HTTP range requests → results client-side ``` ## The dataset id @@ -35,7 +37,7 @@ data//*.xls(x) One identifier ties the whole pipeline together: ``` -data/2017/ → parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ +data/2017/ → parser/configs/2017.yml → db/2017.sqlite3 → /thptqg/2017/ ``` `datasets.json` at the repository root declares the ids once, with the row count @@ -44,7 +46,7 @@ because the assembler is a Go program and the web app is not, and JSON is the only format both parse without a dependency. Presentation — titles, labels, search examples, SQL presets — stays in -`web/src/datasets.js`, keyed by id. That file cross-checks the two: a registry +`web/src/lib/datasets.ts`, keyed by id. That file cross-checks the two: a registry entry with no content, or content for a dataset that was never built, throws at module load rather than rendering a page with no title or a link to a database that does not exist. @@ -168,10 +170,11 @@ total descending. | Concern | Choice | Rationale | | --- | --- | --- | -| Storage | Static SQLite file | No backend; the datasets are frozen | -| Compression | gzip in CI, `DecompressionStream` in the browser | Native API, no extra library | -| WASM hosting | `sql.js.org` CDN | Smaller self-hosted artifact | -| Diacritics search | Pre-computed `ho_ten_ascii` | `LOWER(REPLACE(...))` at query time defeats the index | +| Storage | Static SQLite file, read by range request | No backend; the datasets are frozen, and a lookup needs a few pages of them | +| Compression | None | A byte range of a gzip stream is not a byte range of the database | +| WASM hosting | Bundled with the app | `sql.js-httpvfs` ships its own build; one less third-party runtime dependency | +| Diacritics search | Pre-computed `ho_ten_ascii`, indexed word by word | `LOWER(REPLACE(...))` at query time defeats the index, and `LIKE '%x%'` reads the whole table | +| Row count in the footer | Read from `datasets.json` | `COUNT(*)` scans an index — 20 MB over range requests | | SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes cannot persist; the allowlist prevents confusion | | Row caps | 100 (lookup), 1000 (SQL) | Keeps DOM render sizes reasonable | | Routing | SvelteKit file routes, prerendered | Each dataset gets a real HTML file with its own title | @@ -179,12 +182,16 @@ total descending. ## Risks and limitations -- **Database size.** 45–48 MB gzipped per dataset; slow links wait, mitigated by - a progress bar. -- **Browser memory.** The full database lives in RAM; older mobile devices may - run out. -- **`sql.js.org` dependency.** If that CDN is unreachable, the WASM fails to - load. Self-hosting `sql-wasm.wasm` and updating `SQL_WASM_URL` in - `web/src/lib/sqlite.svelte.ts` is the fix. +- **Unindexed queries are expensive.** The SQL tab can express a query that + walks the table, which over range requests means fetching 100+ MB. A byte + budget stops one before it gets that far, and the tab warns before it opens. +- **`Content-Encoding` breaks everything.** If the host ever compresses + `.sqlite3` on the wire, ranges address compressed bytes and + `sql.js-httpvfs` refuses to open the file. Verify after a deploy: + `curl -sI …/db/2016.sqlite3` must show no `content-encoding`. +- **`sql.js-httpvfs` is unmaintained** (0.8.12, September 2022) and ships its + own SQLite WASM. `sqlite-wasm-http`, on the official build, is the fallback. +- **Hosted size.** 528 MB for both datasets against the 1 GB GitHub Pages + limit; a third dataset of this size would not fit. - **Excel format drift.** A new source file with an unseen header layout needs a new branch in `parser/internal/ingest/detect2016.go` or a new config. diff --git a/parser/internal/schema/schema.go b/parser/internal/schema/schema.go index 95ed9bc..5ad429d 100644 --- a/parser/internal/schema/schema.go +++ b/parser/internal/schema/schema.go @@ -18,9 +18,17 @@ import "regexp" // DDL is executed verbatim after the output database is (re)created. // -// idx_ten_cum_thi is partial, so it holds zero entries on the 2017 dataset — -// where the column is always NULL — while staying useful for the 2016 -// cluster-grouping queries. Partial indexes are SQLite-specific. +// Every index here is chosen for a database read over HTTP range requests, +// where an unindexed query downloads the table. The rules that follow from +// that: +// +// - No index on ho_ten or ho_ten_ascii. Neither substring nor prefix LIKE can +// use one (SQLite's LIKE optimisation needs a NOCASE index or +// case_sensitive_like), so both scanned the whole table. name_word replaces +// them. +// - idx_ten_cum_thi is partial, so it holds zero entries on the 2017 dataset +// — where the column is always NULL — while serving the 2016 cluster +// grouping. Partial indexes are SQLite-specific. // // This text is frozen: it decides the shape of every database the parser // produces. TestDDLIsFrozen holds an independent copy so any edit has to be @@ -50,9 +58,48 @@ CREATE TABLE student ( tieng_nhat REAL, tieng_trung REAL ); -CREATE INDEX idx_ho_ten ON student(ho_ten); -CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; + +CREATE TABLE name_word ( + word TEXT NOT NULL, + so_bao_danh TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + PRIMARY KEY (word, so_bao_danh) +) WITHOUT ROWID; + +CREATE TABLE name_word_freq ( + word TEXT PRIMARY KEY, + n INTEGER NOT NULL +) WITHOUT ROWID; +` + +// PostLoadSQL runs once the student rows are in, before VACUUM. +// +// The frequency table is what lets the site pick which word of a query to seek +// on: the vocabulary is about 4,400 words and the rarest word of a real query +// matches a few hundred rows, so seeking on it and filtering the rest inside +// name_word keeps a search to a few hundred kilobytes. +// +// The three score indexes are partial for the same reason idx_ten_cum_thi is: +// each covers only the exam year that has the column, and each costs about +// 4 MB. They exist so the SQL presets that rank by these columns seek instead +// of scanning 127 MB. +const PostLoadSQL = ` +INSERT INTO name_word_freq (word, n) + SELECT word, COUNT(*) FROM name_word GROUP BY word; +CREATE INDEX idx_toan ON student(toan) WHERE toan IS NOT NULL; +CREATE INDEX idx_khtn ON student(khtn) WHERE khtn IS NOT NULL; +CREATE INDEX idx_khxh ON student(khxh) WHERE khxh IS NOT NULL; +` + +// NameWordInsertSQL adds one row per distinct word of a candidate's ASCII name. +// +// ho_ten_ascii is carried along deliberately: a query with several words seeks +// on the rarest one and filters the others against this copy, so the whole +// match happens inside one b-tree and only the surviving rows are read from +// student. +const NameWordInsertSQL = ` +INSERT OR IGNORE INTO name_word (word, so_bao_danh, ho_ten_ascii) VALUES (?, ?, ?) ` // IdentityFields are the identity columns, in INSERT parameter order. diff --git a/parser/internal/schema/schema_test.go b/parser/internal/schema/schema_test.go index d54f72b..0cbce5c 100644 --- a/parser/internal/schema/schema_test.go +++ b/parser/internal/schema/schema_test.go @@ -109,15 +109,53 @@ CREATE TABLE student ( tieng_nhat REAL, tieng_trung REAL ); -CREATE INDEX idx_ho_ten ON student(ho_ten); -CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL; + +CREATE TABLE name_word ( + word TEXT NOT NULL, + so_bao_danh TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + PRIMARY KEY (word, so_bao_danh) +) WITHOUT ROWID; + +CREATE TABLE name_word_freq ( + word TEXT PRIMARY KEY, + n INTEGER NOT NULL +) WITHOUT ROWID; ` if DDL != want { t.Errorf("DDL changed\n--- got ---\n%s\n--- want ---\n%s", DDL, want) } } +// TestNoIndexOnNameColumns: the databases are read over HTTP range requests, so +// an index that no query can use is dead weight in a file the browser pages +// through. Neither substring nor prefix LIKE can use one on these columns — +// name_word is what serves name search. +func TestNoIndexOnNameColumns(t *testing.T) { + for _, dead := range []string{"idx_ho_ten ", "idx_ho_ten_ascii"} { + if strings.Contains(DDL, dead) { + t.Errorf("DDL creates %q, which no query plan can use", dead) + } + } +} + +// TestPostLoadBuildsTheSearchTables: the frequency table is what lets a search +// pick which word to seek on, and the score indexes are what keep the SQL +// presets off a full scan. +func TestPostLoadBuildsTheSearchTables(t *testing.T) { + for _, want := range []string{ + "INSERT INTO name_word_freq", + "CREATE INDEX idx_toan", + "CREATE INDEX idx_khtn", + "CREATE INDEX idx_khxh", + } { + if !strings.Contains(PostLoadSQL, want) { + t.Errorf("PostLoadSQL is missing %q", want) + } + } +} + // TestScorePatternsMatchScores exercises each pattern against the shape the // DIEM_THI cell actually carries, including the wide runs of spaces seen in the // real corpus. diff --git a/parser/internal/writer/writer.go b/parser/internal/writer/writer.go index 339918e..9f7bc0a 100644 --- a/parser/internal/writer/writer.go +++ b/parser/internal/writer/writer.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/tiennm99/thptqg/parser/internal/schema" "github.com/tiennm99/thptqg/parser/internal/sqlitedb" @@ -110,10 +111,77 @@ type Stats struct { Errors uint64 } -// Finish runs VACUUM and prints the stats block. +// BuildNameIndex fills name_word from the student rows, one entry per distinct +// word of each ASCII name. // -// VACUUM must run AFTER the transaction commits — SQLite refuses it inside one. +// A second pass rather than a write alongside each insert: a repeated exam +// number replaces its earlier row, and the words of the row it replaced would +// otherwise stay behind pointing at a name that is no longer there. +func BuildNameIndex(db *sql.DB) error { + rows, err := db.Query("SELECT so_bao_danh, ho_ten_ascii FROM student") + if err != nil { + return fmt.Errorf("read names: %w", err) + } + defer rows.Close() + + tx, err := db.Begin() + if err != nil { + return fmt.Errorf("begin name index: %w", err) + } + stmt, err := tx.Prepare(schema.NameWordInsertSQL) + if err != nil { + tx.Rollback() + return fmt.Errorf("prepare name index: %w", err) + } + + var words uint64 + seen := make(map[string]struct{}, 8) + for rows.Next() { + var sbd, ascii string + if err := rows.Scan(&sbd, &ascii); err != nil { + tx.Rollback() + return fmt.Errorf("scan name: %w", err) + } + clear(seen) + for _, w := range strings.Fields(ascii) { + if _, dup := seen[w]; dup { + continue + } + seen[w] = struct{}{} + if _, err := stmt.Exec(w, sbd, ascii); err != nil { + tx.Rollback() + return fmt.Errorf("insert name word: %w", err) + } + words++ + } + } + if err := rows.Err(); err != nil { + tx.Rollback() + return fmt.Errorf("read names: %w", err) + } + if err := stmt.Close(); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit name index: %w", err) + } + + if _, err := db.Exec(schema.PostLoadSQL); err != nil { + return fmt.Errorf("post-load statements: %w", err) + } + fmt.Printf("Name index: %d words\n", words) + return nil +} + +// Finish builds the derived tables, runs VACUUM and prints the stats block. +// +// VACUUM must run AFTER the transaction commits — SQLite refuses it inside one +// — and after the name index, so the file is laid out in one pass. func Finish(db *sql.DB, dbPath string, st Stats) error { + if err := BuildNameIndex(db); err != nil { + return err + } if _, err := db.Exec("VACUUM"); err != nil { return fmt.Errorf("vacuum: %w", err) } diff --git a/plans/260814-1200-httpvfs-range-queries/plan.md b/plans/260814-1200-httpvfs-range-queries/plan.md new file mode 100644 index 0000000..427f24b --- /dev/null +++ b/plans/260814-1200-httpvfs-range-queries/plan.md @@ -0,0 +1,71 @@ +# Serve the databases over HTTP range requests + +Status: implemented, unverified in a browser + +The whole-database download is gone. `sql.js-httpvfs` reads the pages a query +touches, so the databases ship raw as `.sqlite3` — a byte range of a gzip +stream is not a byte range of a database. + +## Why the schema had to change first + +Measured on the real 2016 database (223.5 MB before, 4 KB pages, 27 rows/page): + +| Query | Plan before | Would have fetched | +| --- | --- | --- | +| `so_bao_danh = ?` | SEARCH via PK | ~20 KB | +| `ho_ten_ascii LIKE '%x%'` | SCAN | 127 MB | +| `ho_ten_ascii LIKE 'x%'` | SCAN — the LIKE optimisation needs a NOCASE index | 127 MB | +| `COUNT(*)` | covering scan of idx_ho_ten_ascii | 20 MB | +| preset `ORDER BY toan DESC LIMIT 10` | SCAN + temp b-tree | 127 MB | + +So substring search was impossible, prefix search was no better, and the footer +count alone cost 20 MB per page load. + +## What shipped + +**Parser.** `name_word(word, so_bao_danh, ho_ten_ascii)` WITHOUT ROWID — the +table is the index — plus `name_word_freq(word, n)` and partial indexes on +`toan`, `khtn`, `khxh`. Dropped `idx_ho_ten` and `idx_ho_ten_ascii`: no query +plan could use either. + +877,460 names hold 2.87M word entries over a vocabulary of 4,397. A search asks +the frequency table which word is rarest, seeks on that one, and filters the +rest against the `ho_ten_ascii` copy inside the same b-tree — so "buu loc" still +finds "Nguyễn Bửu Lộc", in a few hundred KB. + +| Segment | 2016 | +| --- | --- | +| `student` | 127 MB | +| `name_word` | 97 MB | +| `idx_ten_cum_thi` | 37 MB | +| PK autoindex | 15 MB | +| score indexes | 12 MB | +| **total** | **288.6 MB** (2017: 237.7 MB) | + +**Assembler.** Publishes uncompressed; the size guard reads the raw size; the +stray-artifact check now rejects journals, `.db` and `.gz`. + +**Web.** `RemoteDatabase` wraps `createDbWorker`. The search tab runs with a +25 MB byte budget, the SQL tab asks for consent and then gets 250 MB, and the +bytes fetched are shown next to the query time. The footer count comes from +`datasets.json`. + +## Verified + +- Row counts unchanged: 877,460 and 861,068, both through the assembler guards. +- Every query the app issues is index-driven, checked with `EXPLAIN QUERY PLAN`: + `SEARCH w USING PRIMARY KEY (word>? AND word=0.10.0" } }, - "node_modules/sql.js": { - "version": "1.14.1", - "license": "MIT" + "node_modules/sql.js-httpvfs": { + "version": "0.8.12", + "resolved": "https://registry.npmjs.org/sql.js-httpvfs/-/sql.js-httpvfs-0.8.12.tgz", + "integrity": "sha512-lcEBc2q0psFRfdCx8Di22oUIkkv5MUIaVO/fGCj/Jjx6YQDKVylQEcjd7NSSbmINHTRwVkm/vWP8uuevT7Rkkw==", + "license": "Apache-2.0", + "dependencies": { + "comlink": "^4.3.0" + } }, "node_modules/stackback": { "version": "0.0.2", diff --git a/web/package.json b/web/package.json index a3f8851..fbf0a85 100644 --- a/web/package.json +++ b/web/package.json @@ -12,7 +12,7 @@ "lint": "eslint . && svelte-check --tsconfig ./tsconfig.json" }, "dependencies": { - "sql.js": "^1.14.1" + "sql.js-httpvfs": "^0.8.12" }, "devDependencies": { "@eslint/js": "^9.39.4", @@ -20,7 +20,7 @@ "@sveltejs/kit": "^2.70.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.3.3", - "@types/sql.js": "^1.4.9", + "@types/sql.js": "^1.4.11", "eslint": "^9.39.4", "eslint-plugin-svelte": "^3.14.0", "globals": "^17.4.0", diff --git a/web/src/lib/components/custom-query.svelte b/web/src/lib/components/custom-query.svelte index be44eee..22d0732 100644 --- a/web/src/lib/components/custom-query.svelte +++ b/web/src/lib/components/custom-query.svelte @@ -1,5 +1,5 @@
@@ -97,7 +101,7 @@ type="button" class="btn-chip rounded-md" onclick={() => runPreset(preset.sql)} - {disabled} + disabled={disabled || running} > {preset.label} @@ -111,7 +115,7 @@ class="query-form mb-4" onsubmit={(e) => { e.preventDefault(); - execute(sql); + void execute(sql); }} > -
- {#if execTime !== null} {rows.length} kết quả · {execTime}ms {/if} + {#if db} + + Đã tải: {formatBytes(db.bytesRead)} + {/if}
@@ -145,7 +153,7 @@ [&>th]:bg-surface-alt [&>th]:px-2 [&>th]:py-2.5 [&>th]:text-left [&>th]:whitespace-nowrap" > - {#each columns as col, i (i)} + {#each columns as col (col)} {col} {/each} @@ -155,9 +163,9 @@ - {#each row as cell, ci (ci)} + {#each columns as col (col)} - {cell === null ? "NULL" : String(cell)} + {row[col] === null ? "NULL" : String(row[col])} {/each} diff --git a/web/src/lib/datasets.ts b/web/src/lib/datasets.ts index 4249d19..713c669 100644 --- a/web/src/lib/datasets.ts +++ b/web/src/lib/datasets.ts @@ -25,7 +25,7 @@ const SUBTITLE = "Dữ liệu thí sinh toàn quốc · Hỗ trợ truy vấn SQ * (crawler/internal/sources/source_.go); it is duplicated here because a Go * module and a Vite app cannot share a constant. Keep the two in step. */ -type Content = Omit & { presets: PresetGroup[] }; +type Content = Omit & { presets: PresetGroup[] }; const CONTENT: Record = { 2016: { @@ -67,6 +67,7 @@ for (const id of Object.keys(CONTENT)) { export const DATASETS: Dataset[] = registry.datasets.map((d) => ({ id: d.id, dbSizeMb: d.dbSizeMb, + rows: d.expectedRows, // Derived from expectedRows so the count the hub shows is the same number the // assembler enforces. blurb: `${d.expectedRows.toLocaleString("vi-VN")} thí sinh`, @@ -86,7 +87,12 @@ export function pathOf(dataset: Dataset, base: string): string { return `${base}/${dataset.id}/`; } -/** Gzipped database URL, e.g. dbOf(d, "/thptqg") → "/thptqg/db/2017.db.gz". */ +/** + * Database URL, e.g. dbOf(d, "/thptqg") → "/thptqg/db/2017.sqlite3". + * + * Uncompressed on purpose: the browser reads byte ranges of it, and a range of + * a gzip stream is not a range of the database. + */ export function dbOf(dataset: Dataset, base: string): string { - return `${base}/db/${dataset.id}.db.gz`; + return `${base}/db/${dataset.id}.sqlite3`; } diff --git a/web/src/lib/search.ts b/web/src/lib/search.ts new file mode 100644 index 0000000..76d780b --- /dev/null +++ b/web/src/lib/search.ts @@ -0,0 +1,93 @@ +import { normaliseExamId } from "./query-mode"; +import type { RemoteDatabase } from "./sqlite.svelte"; +import { toAscii } from "./to-ascii"; +import type { Student } from "./types"; + +export const MAX_RESULTS = 100; + +/** + * Name search over the name_word table. + * + * A query is matched word by word, each word as a prefix, in any order — so + * "buu loc" finds "Nguyễn Bửu Lộc". The work is arranged so that only one word + * is ever seeked on and the rest are filtered inside the same b-tree: + * + * 1. ask name_word_freq how many entries each word prefix covers (the whole + * vocabulary is ~4,400 rows, so this is a couple of pages); + * 2. seek on the rarest one — for a real name that is a few hundred to a few + * thousand entries rather than the 300,000 a word like "thi" would walk; + * 3. filter the other words against the ho_ten_ascii copy carried in + * name_word, so nothing is read from student until a row has matched; + * 4. join to student for the rows that survive, at most MAX_RESULTS of them. + * + * Every step is an index seek. A search costs a few hundred KB. + */ +export async function searchByName(db: RemoteDatabase, query: string): Promise { + const words = tokenise(query); + if (words.length === 0) return []; + + const seek = await rarest(db, words); + const others = words.filter((w) => w !== seek); + + // A word matches at a word boundary: the leading space makes the first word + // reachable by the same pattern as the rest. + const filters = others.map(() => `(' ' || w.ho_ten_ascii) LIKE ? ESCAPE '\\'`); + const sql = ` + SELECT s.* FROM name_word w + JOIN student s ON s.so_bao_danh = w.so_bao_danh + WHERE w.word >= ? AND w.word < ?${filters.length ? " AND " + filters.join(" AND ") : ""} + LIMIT ${MAX_RESULTS}`; + + return db.query(sql, [seek, upperBound(seek), ...others.map((w) => `% ${escapeLike(w)}%`)]); +} + +/** Exact lookup by exam number: a primary-key seek, a few pages. */ +export async function lookupExamId(db: RemoteDatabase, id: string): Promise { + return db.query("SELECT * FROM student WHERE so_bao_danh = ? LIMIT ?", [ + normaliseExamId(id), + MAX_RESULTS, + ]); +} + +/** Fold to ASCII and split into words, the same shape name_word was built in. */ +export function tokenise(query: string): string[] { + return toAscii(query).split(/\s+/).filter(Boolean); +} + +/** + * The word whose prefix covers the fewest entries, which is the one worth + * seeking on. One round trip for all of them. + */ +async function rarest(db: RemoteDatabase, words: string[]): Promise { + if (words.length === 1) return words[0]; + + const sql = words + .map(() => "SELECT ? AS word, COALESCE(SUM(n), 0) AS n FROM name_word_freq WHERE word >= ? AND word < ?") + .join(" UNION ALL "); + const params = words.flatMap((w) => [w, w, upperBound(w)]); + + const counts = await db.query<{ word: string; n: number }>(sql, params); + let best = words[0]; + let bestN = Infinity; + for (const { word, n } of counts) { + if (n < bestN) { + best = word; + bestN = n; + } + } + return best; +} + +/** + * The exclusive upper bound of a prefix range. U+FFFF sorts above any character + * that can follow the prefix, which is what turns "starts with" into a range + * the index can seek. + */ +function upperBound(prefix: string): string { + return prefix + "￿"; +} + +/** Escape the LIKE wildcards so a user typing % or _ searches for them. */ +function escapeLike(s: string): string { + return s.replace(/[\\%_]/g, (c) => `\\${c}`); +} diff --git a/web/src/lib/sqlite.svelte.ts b/web/src/lib/sqlite.svelte.ts index 1e70ef8..e00d10f 100644 --- a/web/src/lib/sqlite.svelte.ts +++ b/web/src/lib/sqlite.svelte.ts @@ -1,84 +1,106 @@ -import initSqlJs, { type Database } from "sql.js"; - -// The sql.js engine itself, fetched from the upstream CDN rather than bundled. -// It is a hard runtime dependency: if this URL is unreachable, initSqlJs() -// rejects and no dataset can be opened at all, whatever the database fetch does. -const SQL_WASM_URL = "https://sql.js.org/dist/sql-wasm.wasm"; +import { createDbWorker, type WorkerHttpvfs } from "sql.js-httpvfs"; +import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url"; +import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url"; /** - * A SQLite database loaded from a URL into sql.js, with reactive load state. + * The database is read where it lies. SQLite asks for pages, the virtual file + * system turns each into an HTTP range request, and only the pages a query + * touches ever cross the network — a few hundred KB for a lookup, against the + * 45 MB the whole file used to cost before the first query. * - * The whole file is downloaded and decompressed before the first query: a `.gz` - * URL is inflated in the browser. Nothing streams — sql.js needs the complete - * image in memory. + * That only holds while every query is index-driven. The schema exists for it: + * name_word serves name search, and the score indexes serve the SQL presets. + * An unindexed query walks the table and pulls all 100+ MB of it, which is what + * the byte budget below is for. */ -export class SqliteSource { - db = $state(null); - loading = $state(true); + +// Matches the page size the parser writes, so one request is one page. +const CHUNK_BYTES = 4096; + +/** Generous for indexed work: a name search costs well under 1 MB. */ +export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024; + +/** What the SQL tab gets once the user has accepted the cost of a scan. */ +export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024; + +/** + * One remotely-paged database, with the load state the UI needs. + * + * `budgetBytes` is a hard ceiling for the worker's lifetime: past it a query + * fails instead of quietly downloading the file. Raising it means a new worker, + * which costs only the header pages. + */ +export class RemoteDatabase { + ready = $state(false); error = $state(null); - progress = $state(0); + /** Bytes fetched so far, refreshed after every query. */ + bytesRead = $state(0); - #cancelled = false; + #worker: WorkerHttpvfs | null = null; + #opening: Promise; + #closed = false; - constructor(url: string) { - void this.#load(url); + constructor( + readonly url: string, + readonly budgetBytes: number = SEARCH_BUDGET_BYTES, + ) { + this.#opening = this.#open(); } - async #load(url: string) { + async #open(): Promise { try { - const SQL = await initSqlJs({ locateFile: () => SQL_WASM_URL }); - - const response = await fetch(url); - if (!response.ok) throw new Error(`Failed to fetch database: ${response.status}`); - - const contentLength = Number(response.headers.get("Content-Length")) || 0; - const reader = response.body!.getReader(); - const chunks: Uint8Array[] = []; - let received = 0; - - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - received += value.length; - if (contentLength > 0) { - this.progress = Math.round((received / contentLength) * 100); - } - } - if (this.#cancelled) return; - - const blob = new Blob(chunks as BlobPart[]); - const bytes = url.endsWith(".gz") - ? await new Response(blob.stream().pipeThrough(new DecompressionStream("gzip"))).arrayBuffer() - : await blob.arrayBuffer(); - if (this.#cancelled) return; - - this.db = new SQL.Database(new Uint8Array(bytes)); - this.loading = false; + const worker = await createDbWorker( + [{ from: "inline", config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES } }], + workerUrl, + wasmUrl, + this.budgetBytes, + ); + if (this.#closed) throw new Error("closed"); + this.#worker = worker; + this.ready = true; + return worker; } catch (err) { - if (this.#cancelled) return; - this.error = err instanceof Error ? err.message : String(err); - this.loading = false; + if (!this.#closed) { + this.error = message(err); + this.ready = false; + } + throw err; } } - /** Release the database. Call from the owning component's teardown. */ + /** Run a query and return its rows as objects. */ + async query(sql: string, params: unknown[] = []): Promise { + const worker = this.#worker ?? (await this.#opening); + // Comlink erases the generic when it proxies the method across the worker + // boundary, so the row type is asserted here rather than inferred. + const run = worker.db.query as unknown as (sql: string, ...params: unknown[]) => Promise; + try { + return await run(sql, ...params); + } finally { + // Comlink proxies the property, so this is a round trip; worth it because + // the number is the only honest feedback about what a query cost. + this.bytesRead = await worker.worker.bytesRead; + } + } + + /** + * Drop this database. createDbWorker owns the Worker and exposes no handle to + * it, so the thread outlives this call; a page creates at most one per + * dataset and one more if the SQL budget is raised, which is why that is + * tolerable rather than a leak worth working around. + */ close() { - this.#cancelled = true; - this.db?.close(); - this.db = null; + this.#closed = true; + this.#worker = null; + this.ready = false; } } -/** Run a statement and return its rows as objects. */ -export function queryRows(db: Database, sql: string, params?: Record): T[] { - const stmt = db.prepare(sql); - try { - if (params) stmt.bind(params as never); - const rows: T[] = []; - while (stmt.step()) rows.push(stmt.getAsObject() as T); - return rows; - } finally { - stmt.free(); - } +/** True when a query failed because it would have exceeded the byte budget. */ +export function isBudgetError(err: unknown): boolean { + return /maxBytesToRead|too much data|exceeded/i.test(message(err)); +} + +function message(err: unknown): string { + return err instanceof Error ? err.message : String(err); } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index ecf82f3..91eb320 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -35,7 +35,12 @@ export type Student = { /** A dataset as the interface needs it: registry facts plus presentation. */ export type Dataset = { id: string; + /** Size of the hosted database. Nothing downloads it whole; it is shown so a + * user knows what they are querying into. */ dbSizeMb: number; + /** Row count from the registry, so the footer never runs COUNT(*) — that + * scans an index and would cost 20 MB over range requests. */ + rows: number; blurb: string; label: string; title: string; diff --git a/web/src/routes/[dataset]/+page.svelte b/web/src/routes/[dataset]/+page.svelte index ce0720b..ac18283 100644 --- a/web/src/routes/[dataset]/+page.svelte +++ b/web/src/routes/[dataset]/+page.svelte @@ -8,59 +8,47 @@ import SearchForm from "$lib/components/search-form.svelte"; import StudentDetail from "$lib/components/student-detail.svelte"; import { dbOf } from "$lib/datasets"; - import { isExamId, normaliseExamId } from "$lib/query-mode"; - import { SqliteSource, queryRows } from "$lib/sqlite.svelte"; - import { isAsciiOnly, toAscii } from "$lib/to-ascii"; + import { isExamId } from "$lib/query-mode"; + import { MAX_RESULTS, lookupExamId, searchByName } from "$lib/search"; + import { PLAYGROUND_BUDGET_BYTES, RemoteDatabase, SEARCH_BUDGET_BYTES } from "$lib/sqlite.svelte"; import type { Student } from "$lib/types"; - const MAX_RESULTS = 100; - let { data } = $props(); const dataset = $derived(data.dataset); - let source = $state(null); + let db = $state(null); let results = $state(null); let searchError = $state(null); let activeTab = $state<"search" | "sql">("search"); - let totalCount = $state(null); + let sqlWarningOpen = $state(false); + // Raised once the user has accepted that a hand-written query may fetch a lot. + let budget = $state(SEARCH_BUDGET_BYTES); // Owned here, not in SearchForm, so it can be bound to the URL both ways. // The query string is unreadable while prerendering — there is no request — // so a deep link is picked up on the client only. let query = $state(browser ? (page.url.searchParams.get("q") ?? "") : ""); - const db = $derived(source?.db ?? null); - const loading = $derived(source?.loading ?? true); - const loadError = $derived(source?.error ?? null); - const progress = $derived(source?.progress ?? 0); - const busy = $derived(loading || !!loadError); + const opening = $derived(db !== null && !db.ready && db.error === null); + const loadError = $derived(db?.error ?? null); + const busy = $derived(!db?.ready); - // The database is per dataset, and only ever fetched in the browser: $effect - // does not run while prerendering. + // Opened in the browser only: $effect does not run while prerendering. A new + // budget means a new worker, which costs only the header pages. $effect(() => { - const opened = new SqliteSource(dbOf(dataset, base)); - source = opened; + const opened = new RemoteDatabase(dbOf(dataset, base), budget); + db = opened; return () => { opened.close(); - source = null; - results = null; - totalCount = null; + db = null; }; }); - // Candidate count for the footer. One-shot per database: it cannot change - // without a new one. - $effect(() => { - if (!db) return; - const [row] = queryRows<{ c: number }>(db, "SELECT COUNT(*) AS c FROM student"); - totalCount = row?.c ?? null; - }); - - // Hydrate a ?q= deep link as soon as the database is ready. + // Hydrate a ?q= deep link as soon as the database is open. let hydrated = false; $effect(() => { - if (!db || hydrated) return; + if (!db?.ready || hydrated) return; hydrated = true; - if (query) search(query); + if (query) void search(query); }); // Sync the query to ?q= without adding a history entry, so back still leaves @@ -73,35 +61,15 @@ replaceState(q ? `${route}?q=${encodeURIComponent(q)}` : route, page.state); } - function search(q: string) { - if (!db) return; + async function search(q: string) { + const source = db; + if (!source?.ready) return; searchError = null; query = q; writeUrlQuery(q); try { - if (isExamId(q)) { - // Letter-prefixed 2016 IDs are stored upper-case; digits are unaffected. - results = queryRows( - db, - "SELECT * FROM student WHERE so_bao_danh = $q LIMIT $limit", - { $q: normaliseExamId(q), $limit: MAX_RESULTS }, - ); - } else if (isAsciiOnly(q)) { - results = queryRows( - db, - "SELECT * FROM student WHERE ho_ten_ascii LIKE $q LIMIT $limit", - { $q: `%${toAscii(q)}%`, $limit: MAX_RESULTS }, - ); - } else { - results = queryRows( - db, - `SELECT * FROM student - WHERE ho_ten LIKE $q OR ho_ten_ascii LIKE $qn - LIMIT $limit`, - { $q: `%${q}%`, $qn: `%${toAscii(q)}%`, $limit: MAX_RESULTS }, - ); - } + results = isExamId(q) ? await lookupExamId(source, q) : await searchByName(source, q); } catch (err) { searchError = err instanceof Error ? err.message : String(err); } @@ -113,9 +81,34 @@ writeUrlQuery(""); } + function openSqlTab() { + if (budget >= PLAYGROUND_BUDGET_BYTES) { + activeTab = "sql"; + return; + } + sqlWarningOpen = true; + } + + function acceptSqlWarning() { + sqlWarningOpen = false; + // Reopening with the larger budget throws away the current worker, and with + // it the pages it had cached — a few hundred KB, refetched on demand. + budget = PLAYGROUND_BUDGET_BYTES; + activeTab = "sql"; + } + + function declineSqlWarning() { + sqlWarningOpen = false; + activeTab = "search"; + } + // Global shortcuts: Ctrl+Enter submits the SQL query, "/" focuses the search // box unless the user is already typing somewhere. function onKeydown(event: KeyboardEvent) { + if (event.key === "Escape" && sqlWarningOpen) { + declineSqlWarning(); + return; + } if (event.ctrlKey && event.key === "Enter" && activeTab === "sql") { document.querySelector(".query-form")?.requestSubmit(); return; @@ -151,18 +144,8 @@
- {#if loading} -
-

- Đang tải cơ sở dữ liệu ~{dataset.dbSizeMb} MB{progress > 0 ? ` · ${progress}%` : ""} -

-
-
-
-

- Lần đầu có thể mất 10-30 giây. Sau đó trình duyệt sẽ lưu cache và mở nhanh hơn. -

-
+ {#if opening} +

Đang mở cơ sở dữ liệu…

{/if} {#if loadError} @@ -170,20 +153,30 @@ {/if}
- {#each [{ id: "search", label: "Tra cứu" }, { id: "sql", label: "Truy vấn SQL" }] as const as tab (tab.id)} - - {/each} + +
{#if activeTab === "search"} @@ -215,16 +208,44 @@ {/if}
-
+

Nguồn: {dataset.source} - {#if totalCount !== null} - · {totalCount.toLocaleString("vi-VN")} thí sinh - {/if} - · Dữ liệu chỉ mang tính tham khảo + · {dataset.rows.toLocaleString("vi-VN")} thí sinh · Dữ liệu chỉ mang tính tham khảo

+ +{#if sqlWarningOpen} + + +{/if} From dbf13d009494b4192fd5d56e09beb85cccc3843d Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 14 Aug 2026 13:38:44 +0700 Subject: [PATCH 2/5] perf(parser): write the databases with 1 KiB pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The browser fetches this file one page per HTTP request, so the page size is the granularity of every read. At SQLite's 4 KiB default a row reached by an index seek dragged 4 KB across the network; at 1 KiB it drags 1 KB. A name search returns up to 100 scattered rows, so its row fetches fall from about 400 KB to about 100 KB. Measured on the rebuilt 2016 file: 6.3 rows share a page where 27 did. The index walks are sequential and unaffected in bytes — the library's read-ahead already collapses those into few requests. Cost is 4% file size: 2016 288.6 -> 302.4 MB, 2017 237.7 -> 247.3 MB, the site 528 -> 552 MB against the 1 GB GitHub Pages limit. Both sql.js-httpvfs and sqlite-wasm-http recommend this page size. The PRAGMA has to run before the DDL, since a page size is fixed once a table exists, and requestChunkSize on the client has to match or every page read spans two requests. Row counts unchanged and through the assembler guards; query plans re-checked and still index-driven on the rebuilt files. --- datasets.json | 4 +- docs/data-pipeline.md | 4 + docs/deployment-guide.md | 4 +- docs/system-architecture.md | 3 +- parser/internal/writer/writer.go | 13 ++ .../260814-1200-httpvfs-range-queries/plan.md | 16 +- ...-1317-httpvfs-page-size-adoption-report.md | 78 +++++++++ ...0814-1317-httpvfs-best-practices-report.md | 165 ++++++++++++++++++ web/src/lib/sqlite.svelte.ts | 6 +- 9 files changed, 280 insertions(+), 13 deletions(-) create mode 100644 plans/reports/from-research-to-implementation-brainstorm-260814-1317-httpvfs-page-size-adoption-report.md create mode 100644 plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md diff --git a/datasets.json b/datasets.json index fef7161..4543006 100644 --- a/datasets.json +++ b/datasets.json @@ -20,12 +20,12 @@ { "id": "2016", "expectedRows": 877460, - "dbSizeMb": 289 + "dbSizeMb": 302 }, { "id": "2017", "expectedRows": 861068, - "dbSizeMb": 238 + "dbSizeMb": 247 } ] } diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index d199eda..1bb9567 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -181,6 +181,10 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what ## Expected row counts +The databases are written with 1 KiB pages (`PRAGMA page_size` in +`parser/internal/writer/writer.go`) because the browser fetches them a page per +HTTP request. `CHUNK_BYTES` in `web/src/lib/sqlite.svelte.ts` must match. + | id | Source rows | Skipped | DB rows | | --- | --- | --- | --- | | `2016` | 877,460 | 0 | **877,460** | diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 0cf3049..b34a9dc 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -83,8 +83,8 @@ artifact — one missing line away from publishing it. files committed to a repository; the databases are built in CI and uploaded as a Pages artifact, and the documented Pages limits are a 1 GB published site and 100 GB/month of bandwidth, with no per-file figure. The two - databases are 289 MB and 238 MB. -- **Total artifact is about 528 MB**, inside the 1 GB site limit but with less + databases are 302 MB and 247 MB. +- **Total artifact is about 552 MB**, inside the 1 GB site limit but with less headroom than before: a third dataset of this size would not fit. The fallback is `sql.js-httpvfs`'s chunked mode, which splits a database into parts. - **The server must not compress the databases.** Ranges of a compressed body diff --git a/docs/system-architecture.md b/docs/system-architecture.md index d57b38d..850804a 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -175,6 +175,7 @@ total descending. | WASM hosting | Bundled with the app | `sql.js-httpvfs` ships its own build; one less third-party runtime dependency | | Diacritics search | Pre-computed `ho_ten_ascii`, indexed word by word | `LOWER(REPLACE(...))` at query time defeats the index, and `LIKE '%x%'` reads the whole table | | Row count in the footer | Read from `datasets.json` | `COUNT(*)` scans an index — 20 MB over range requests | +| Page size | 1 KiB, matched by `requestChunkSize` | One HTTP request is one page; a row fetched by seek costs 1 KB rather than 4 KB, for about 5% more file | | SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes cannot persist; the allowlist prevents confusion | | Row caps | 100 (lookup), 1000 (SQL) | Keeps DOM render sizes reasonable | | Routing | SvelteKit file routes, prerendered | Each dataset gets a real HTML file with its own title | @@ -191,7 +192,7 @@ total descending. `curl -sI …/db/2016.sqlite3` must show no `content-encoding`. - **`sql.js-httpvfs` is unmaintained** (0.8.12, September 2022) and ships its own SQLite WASM. `sqlite-wasm-http`, on the official build, is the fallback. -- **Hosted size.** 528 MB for both datasets against the 1 GB GitHub Pages +- **Hosted size.** 552 MB for both datasets against the 1 GB GitHub Pages limit; a third dataset of this size would not fit. - **Excel format drift.** A new source file with an unseen header layout needs a new branch in `parser/internal/ingest/detect2016.go` or a new config. diff --git a/parser/internal/writer/writer.go b/parser/internal/writer/writer.go index 9f7bc0a..eb8ea86 100644 --- a/parser/internal/writer/writer.go +++ b/parser/internal/writer/writer.go @@ -43,6 +43,19 @@ func OpenDB(dbPath string) (*sql.DB, error) { if err != nil { return nil, fmt.Errorf("open db %s: %w", dbPath, err) } + // Before the DDL, because a page size cannot change once a table exists — + // only the VACUUM in Finish could rewrite it, and only to this same value. + // + // 1 KiB rather than SQLite's 4 KiB default because the browser reads this + // file a page at a time over HTTP: a row fetched by index seek costs one + // page, so a search that returns 100 scattered rows transfers 100 KB + // instead of 400 KB. It costs about 5% file size, and both sql.js-httpvfs + // and sqlite-wasm-http recommend it. web/src/lib/sqlite.svelte.ts must + // request the same size. + if _, err := db.Exec("PRAGMA page_size = 1024"); err != nil { + db.Close() + return nil, fmt.Errorf("set page size: %w", err) + } if _, err := db.Exec(schema.DDL); err != nil { db.Close() return nil, fmt.Errorf("execute DDL: %w", err) diff --git a/plans/260814-1200-httpvfs-range-queries/plan.md b/plans/260814-1200-httpvfs-range-queries/plan.md index 427f24b..2ba55e2 100644 --- a/plans/260814-1200-httpvfs-range-queries/plan.md +++ b/plans/260814-1200-httpvfs-range-queries/plan.md @@ -35,12 +35,16 @@ finds "Nguyễn Bửu Lộc", in a few hundred KB. | Segment | 2016 | | --- | --- | -| `student` | 127 MB | -| `name_word` | 97 MB | -| `idx_ten_cum_thi` | 37 MB | -| PK autoindex | 15 MB | -| score indexes | 12 MB | -| **total** | **288.6 MB** (2017: 237.7 MB) | +| `student` | 137.5 MB | +| `name_word` | 98.7 MB | +| `idx_ten_cum_thi` | 38.1 MB | +| PK autoindex | 15.3 MB | +| `idx_toan` | 12.6 MB | +| **total** | **302.4 MB** (2017: 247.3 MB) | + +Written with 1 KiB pages, so a row reached by an index seek costs one 1 KB +request instead of 4 KB: 6.3 rows share a page rather than 27, which is what +turns a 100-row search from ~400 KB of row fetches into ~100 KB. **Assembler.** Publishes uncompressed; the size guard reads the raw size; the stray-artifact check now rejects journals, `.db` and `.gz`. diff --git a/plans/reports/from-research-to-implementation-brainstorm-260814-1317-httpvfs-page-size-adoption-report.md b/plans/reports/from-research-to-implementation-brainstorm-260814-1317-httpvfs-page-size-adoption-report.md new file mode 100644 index 0000000..070546a --- /dev/null +++ b/plans/reports/from-research-to-implementation-brainstorm-260814-1317-httpvfs-page-size-adoption-report.md @@ -0,0 +1,78 @@ +# Brainstorm: which httpvfs best practices to adopt + +2026-08-14 13:17. Follows +[the research report](./web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md). +Branch `feat/httpvfs-range-queries`. + +## Problem + +Research surfaced three upstream practices we do not follow. Decide which are +worth the change, knowing nothing can be verified in a browser on this machine. + +## Codebase context (scout) + +| Concern | Touch points | +| --- | --- | +| Page size | `parser/internal/writer/writer.go:41-47` (PRAGMA must precede the DDL — page size is fixed once a table exists), `:185` (VACUUM already applies it), `web/src/lib/sqlite.svelte.ts:18,53` | +| Chunked mode | `databases.go:41` Extension, size guard, **`Clean()` deletes anything not `.sqlite3` — would eat every chunk**, `site.go:135-138,153`, `datasets.ts:91-97`, `datasets.json`, ~15 tests | +| Library swap | `sqlite.svelte.ts:1-3,50-58,76,82`, `package.json:15`; all consumers go through `RemoteDatabase`, so the blast radius is one file | + +## Options evaluated + +### A. page_size 1024 + requestChunkSize 1024 — ADOPTED + +- Upstream consensus: phiresky and mmomtchev both recommend 1024. +- Honest sizing for *our* pattern: row fetches 400 KB → 100 KB per search; + the index walk is sequential so bytes are unchanged and only the request + count rises, which prefetch read-heads collapse. Net ≈ 300 KB saved per + search — bandwidth, not latency. +- Cost: ~10 lines; file size +5% (528 → ~555 MB total, still under the 1 GB + Pages limit); both databases rebuilt. + +### B. serverMode chunked — REJECTED + +- Only benefit is CDN cache efficiency. GitHub Pages serves everything with + `Cache-Control: max-age=600`, and each deploy relays out SQLite pages anyway, + so cross-deploy caching is zero either way. +- Cost: split step, config JSON, `Clean()`/guards/`dbOf()`/`datasets.json` + rework, ~15 tests. +- Complexity buying a benefit the host cancels. Revisit only behind a CDN with + long TTLs. + +### C. swap to sqlite-wasm-http — DEFERRED + +- For: maintained (Dec 2025), official SQLite WASM instead of a 2022 fork; + matches this repo's posture on stale dependencies. Swap is one file. +- Against: its differentiator (shared cache) needs COOP/COEP headers GitHub + Pages cannot send, so we would get the synchronous fallback and the + maintenance benefit only. And the current integration has never run in a + browser — swapping now means two unverified variables and no way to tell + which broke. +- Revisit after the current build is verified live. + +## Decision + +Adopt **A only**. + +## Implementation notes + +1. `PRAGMA page_size = 1024` in `writer.OpenDB`, between `sql.Open` and + `db.Exec(schema.DDL)`. The existing VACUUM in `Finish` applies it. +2. `CHUNK_BYTES = 1024` in `web/src/lib/sqlite.svelte.ts`; it feeds + `requestChunkSize` and must equal the page size. +3. Rebuild both databases, update `dbSizeMb` in `datasets.json` to the new + sizes, re-run the assembler guards. + +## Risks + +- The benefit is arithmetic plus upstream authority, not measurement. The byte + counter in the SQL tab is the check, once deployed. +- Prefetch deliberately overfetches ahead of the cursor, so the 25 MB search + budget may trip earlier than a strict page count suggests. Tune after a real + measurement, not before. + +## Unresolved questions + +1. Does 1024 actually beat 4096 for our queries in a browser? +2. Is Fastly's caching of ranges over a 300 MB object good enough that chunked + mode stays unnecessary? diff --git a/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md b/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md new file mode 100644 index 0000000..ef11e7c --- /dev/null +++ b/plans/reports/web-sqlite-range-hosting-research-260814-1317-httpvfs-best-practices-report.md @@ -0,0 +1,165 @@ +# Research Report: sql.js-httpvfs best practices + +Conducted 2026-08-14 13:17 (Asia/Saigon). Context: two static SQLite files +(2016 = 288.6 MB, 2017 = 237.7 MB) on GitHub Pages, branch +`feat/httpvfs-range-queries`. + +## Executive summary + +Our implementation matches upstream guidance on the thing that matters most — +index design — and diverges on one measurable parameter: **page size**. Both +phiresky (sql.js-httpvfs) and mmomtchev (sqlite-wasm-http) recommend +`page_size = 1024`; we shipped 4096. For our access pattern (~100 scattered +single-row reads per search) that is a real 4× overfetch on the row-fetch half +of a query. + +Two findings reduce risk rather than add work. Prefetching with three virtual +read heads makes a sequential scan cost a *logarithmic* number of requests, so +our byte estimates hold but latency is better than assumed. And the canonical +demo hosts a **670 MiB** database on GitHub Pages, so 289 MB is precedented. + +One finding is new and worth a decision: **chunked mode** (split file + JSON +config) exists specifically to make CDN caching effective for large databases, +which matters because GitHub Pages serves everything with `Cache-Control: +max-age=600`. + +## Methodology + +- Sources: 5 (1 primary blog, 2 project READMEs, 1 recent practitioner + writeup, 1 search on Pages/Fastly caching), plus direct reading of the + installed `sql.js-httpvfs@0.8.12` bundle in a previous session. +- Date range: 2021 (canonical post) → March 2026 (practitioner writeup). +- Gemini CLI absent → WebSearch/WebFetch. + +## Key findings + +### 1. Page size: recommended 1024, we use 4096 + +Both projects say the same thing. phiresky set 1 KiB pages "to balance request +overhead against bandwidth efficiency"; sqlite-wasm-http says "it is highly +recommended to decrease your SQLite page size to 1024 bytes for maximum +performance" (`PRAGMA page_size=1024; VACUUM`). + +`requestChunkSize` must match the page size. + +Measured on our 2016 file *before* the schema change: + +| page_size | file size | +| --- | --- | +| 1024 | 235.8 MB | +| 4096 | 223.5 MB | +| 8192 | 221.8 MB | + +So 1024 costs ~5.5% file size. What it buys: a scattered row read fetches 1 KB +instead of 4 KB. Our search does ~100 of those, so the row-fetch half of a +search drops from ~400 KB to ~100 KB. The index-walk half is sequential and +benefits from prefetch either way. + +**Verdict: switch to 1024 + `requestChunkSize: 1024`.** Our workload is +dominated by scattered single-row reads, which is exactly the case small pages +serve. + +### 2. Prefetch changes request count, not bytes + +"Three separate virtual read heads" detect sequential access and grow request +sizes exponentially, so "index scans or table scans reading more than a few KiB +of data will only cause a number of requests that is logarithmic in the total +byte length." + +Consequence for our analysis: a full table scan still transfers ~127 MB (bytes +are bytes), but in tens of requests rather than tens of thousands. Our byte +budget is the right guardrail; a request-count budget would not be. + +### 3. Index design — we already comply + +- Covering indexes: put every column the query needs *in* the index, else + SQLite does "another random access (unpredictable) read and thus HTTP request + to retrieve the actual value for every data point". This is exactly why + `name_word` carries `ho_ten_ascii`. +- Column order decides which lookups are cheap. +- Verify with `EXPLAIN QUERY PLAN`; a `SCAN` means the whole table crosses the + network. We did this for every query the app issues. + +### 4. Chunked mode exists for CDN caching + +`serverMode: "chunked"` splits the database into parts (10 MB is the commonly +cited size) with a JSON config. Stated benefit: "CDN caching much more +effective" for large databases. + +Relevant because **GitHub Pages sets `Cache-Control: max-age=600`** — ten +minutes — on everything. Range responses are cached by Fastly per object; with +one 289 MB object the practical caching story is weaker than with 29 chunks +that a CDN edge can hold whole. Chunked mode also sidesteps any future per-file +concern. + +Cost: a build step to split files + emit config, and every deploy invalidates +all chunks anyway (SQLite page layout is not deterministic across rebuilds). + +### 5. Hosting facts confirmed + +- Range requests work on Pages "out of the box" — matches our own probe (206 + + correct `Content-Range`). +- CORS headers (`Access-Control-Allow-Origin`, `Access-Control-Allow-Headers: + Range`) only matter cross-origin. Ours is same-origin — non-issue. +- 670 MiB database on Pages is the canonical demo. 289 MB is not exotic. +- `Content-Encoding` remains the one fatal case: the library discards + `Content-Length` and throws when a HEAD carries a non-identity encoding. + Unknown extensions like `.sqlite3` are served `application/octet-stream` and + left alone. + +### 6. Alternatives + +| | sql.js-httpvfs | sqlite-wasm-http | +| --- | --- | --- | +| WASM base | own sql.js fork (~3.36 era) | official `@sqlite.org/sqlite-wasm` | +| Last release | 0.8.12, Sept 2022 | 1.2.0, Dec 2023; activity into Dec 2025 | +| Self-description | "demo-level code… not for high stability" | "experimental" | +| Concurrency | one worker | multiple connections, shared cache | +| Shared cache needs | — | `SharedArrayBuffer` → COOP/COEP headers | +| On GitHub Pages | works | works, but **Pages cannot set COOP/COEP**, so it falls back to the synchronous backend without shared cache | +| Module format | CJS+ESM | **ES6 only** | + +Both are self-declared experimental. sqlite-wasm-http's advantage (maintained, +official WASM) is real; its headline feature (shared cache) is unavailable on +Pages precisely because Pages cannot send cross-origin isolation headers. + +## Implementation recommendations + +1. **Change page size to 1024 and `requestChunkSize` to 1024.** Parser sets + `PRAGMA page_size=1024` before DDL; VACUUM already runs. Cost ~+5% file + size; benefit ~4× less overfetch per row read. +2. **Keep `serverMode: "full"` for now.** Chunked mode's benefit is CDN cache + efficiency, which Pages' 10-minute TTL blunts. Revisit if measured repeat- + visit cost is bad. +3. **Keep sql.js-httpvfs.** Switching to sqlite-wasm-http buys a maintained + dependency but loses nothing we use, and its differentiator does not work on + Pages. Note it as the escape hatch. +4. **Verify `Content-Encoding` after deploy** — the single fatal hosting case. +5. Keep the byte budget; drop any idea of a request-count budget. + +## Common pitfalls + +- Unindexed query → whole table over the network. `EXPLAIN QUERY PLAN` is the + check. +- Page size mismatched with `requestChunkSize` → every logical page read spans + two requests. +- Serving the database compressed → library refuses to open it. +- Assuming CDN caching helps: on Pages, `max-age=600`. + +## References + +- [Hosting SQLite databases on GitHub Pages — phiresky](https://phiresky.github.io/blog/2021/hosting-sqlite-databases-on-github-pages/) +- [phiresky/sql.js-httpvfs](https://github.com/phiresky/sql.js-httpvfs) +- [mmomtchev/sqlite-wasm-http](https://github.com/mmomtchev/sqlite-wasm-http) +- [Query SQLite on GitHub Pages with sql.js-httpvfs (Mar 2026)](https://recca0120.github.io/en/2026/03/07/sql-js-httpvfs-static-hosting/) +- [sqlite3 WebAssembly documentation](https://sqlite.org/wasm) +- [GitHub Pages asset caching discussion](https://github.com/orgs/community/discussions/11884) + +## Unresolved questions + +1. Does 1024 measurably beat 4096 *for our queries*? Only a browser with the + byte counter can answer; the estimate says yes for row fetches. +2. Does Fastly cache 206 responses for a 289 MB object well enough that chunked + mode is unnecessary? Needs a deployed measurement. +3. Does the read-head prefetch overfetch on our index-range walks (fetching + ahead beyond `LIMIT 100`)? Unknown without instrumentation. diff --git a/web/src/lib/sqlite.svelte.ts b/web/src/lib/sqlite.svelte.ts index e00d10f..5ef0f2f 100644 --- a/web/src/lib/sqlite.svelte.ts +++ b/web/src/lib/sqlite.svelte.ts @@ -14,8 +14,10 @@ import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url"; * the byte budget below is for. */ -// Matches the page size the parser writes, so one request is one page. -const CHUNK_BYTES = 4096; +// Must equal the page size the parser writes (PRAGMA page_size in +// parser/internal/writer/writer.go), so one request is exactly one page. A +// mismatch makes every logical page read span two requests. +const CHUNK_BYTES = 1024; /** Generous for indexed work: a name search costs well under 1 MB. */ export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024; From d7e2fa2d12ed3bf4472fb7a77bb2e2b4ab586e6b Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 14 Aug 2026 13:43:11 +0700 Subject: [PATCH 3/5] feat(web): trace what each query costs over the network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole design rests on a claim nobody has measured: that a lookup fetches a few hundred KB rather than the file. Every query now reports what it actually cost. [httpvfs] name "nguyen buu loc" seeking "buu" — 14 request(s), 78 KB, 212 ms · session 31 request(s), 180 KB of 302.4 MB The numbers come from the worker's getStats() rather than its bytesRead counter: bytesRead is the budget accumulator and resets itself to zero the moment a query trips the ceiling, so it would under-report exactly when the number matters. getStats() reports per-file totals including what the read heads prefetched, which is the honest figure — prefetch overfetch is invisible to a page count. Opening a database logs too, since the header and schema pages are read before any query and would otherwise inflate the first search. The SQL tab shows the same pair on screen: this query, then the session. --- web/src/lib/components/custom-query.svelte | 16 ++--- web/src/lib/search.ts | 17 ++++-- web/src/lib/sqlite.svelte.ts | 71 +++++++++++++++++++--- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/web/src/lib/components/custom-query.svelte b/web/src/lib/components/custom-query.svelte index 22d0732..d18df0b 100644 --- a/web/src/lib/components/custom-query.svelte +++ b/web/src/lib/components/custom-query.svelte @@ -1,5 +1,5 @@
@@ -133,9 +130,12 @@ {#if execTime !== null} {rows.length} kết quả · {execTime}ms {/if} - {#if db} - - Đã tải: {formatBytes(db.bytesRead)} + {#if db?.lastCost} + + + Truy vấn này: {db.lastCost.requests} yêu cầu · {formatBytes(db.lastCost.bytes)} + · Phiên: {db.requests} yêu cầu · {formatBytes(db.bytesRead)} + {/if}
diff --git a/web/src/lib/search.ts b/web/src/lib/search.ts index 76d780b..b40cf03 100644 --- a/web/src/lib/search.ts +++ b/web/src/lib/search.ts @@ -38,15 +38,20 @@ export async function searchByName(db: RemoteDatabase, query: string): Promise= ? AND w.word < ?${filters.length ? " AND " + filters.join(" AND ") : ""} LIMIT ${MAX_RESULTS}`; - return db.query(sql, [seek, upperBound(seek), ...others.map((w) => `% ${escapeLike(w)}%`)]); + return db.query( + sql, + [seek, upperBound(seek), ...others.map((w) => `% ${escapeLike(w)}%`)], + `name ${JSON.stringify(words.join(" "))} seeking ${JSON.stringify(seek)}`, + ); } /** Exact lookup by exam number: a primary-key seek, a few pages. */ export async function lookupExamId(db: RemoteDatabase, id: string): Promise { - return db.query("SELECT * FROM student WHERE so_bao_danh = ? LIMIT ?", [ - normaliseExamId(id), - MAX_RESULTS, - ]); + return db.query( + "SELECT * FROM student WHERE so_bao_danh = ? LIMIT ?", + [normaliseExamId(id), MAX_RESULTS], + `exam id ${JSON.stringify(normaliseExamId(id))}`, + ); } /** Fold to ASCII and split into words, the same shape name_word was built in. */ @@ -66,7 +71,7 @@ async function rarest(db: RemoteDatabase, words: string[]): Promise { .join(" UNION ALL "); const params = words.flatMap((w) => [w, w, upperBound(w)]); - const counts = await db.query<{ word: string; n: number }>(sql, params); + const counts = await db.query<{ word: string; n: number }>(sql, params, "word frequencies"); let best = words[0]; let bestN = Infinity; for (const { word, n } of counts) { diff --git a/web/src/lib/sqlite.svelte.ts b/web/src/lib/sqlite.svelte.ts index 5ef0f2f..f8e33d5 100644 --- a/web/src/lib/sqlite.svelte.ts +++ b/web/src/lib/sqlite.svelte.ts @@ -1,4 +1,4 @@ -import { createDbWorker, type WorkerHttpvfs } from "sql.js-httpvfs"; +import { createDbWorker, type SqliteStats, type WorkerHttpvfs } from "sql.js-httpvfs"; import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url"; import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url"; @@ -25,6 +25,9 @@ export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024; /** What the SQL tab gets once the user has accepted the cost of a scan. */ export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024; +/** What one query cost over the network. */ +export type QueryCost = { requests: number; bytes: number; ms: number }; + /** * One remotely-paged database, with the load state the UI needs. * @@ -35,12 +38,18 @@ export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024; export class RemoteDatabase { ready = $state(false); error = $state(null); - /** Bytes fetched so far, refreshed after every query. */ + /** Bytes fetched by this database so far, prefetch included. */ bytesRead = $state(0); + /** HTTP range requests issued so far. */ + requests = $state(0); + /** What the most recent query cost on its own. */ + lastCost = $state(null); #worker: WorkerHttpvfs | null = null; #opening: Promise; #closed = false; + // Previous cumulative reading, so a query's own cost is a subtraction. + #seen = { requests: 0, bytes: 0 }; constructor( readonly url: string, @@ -50,6 +59,7 @@ export class RemoteDatabase { } async #open(): Promise { + const opened = performance.now(); try { const worker = await createDbWorker( [{ from: "inline", config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES } }], @@ -60,6 +70,9 @@ export class RemoteDatabase { if (this.#closed) throw new Error("closed"); this.#worker = worker; this.ready = true; + // Opening is not free either: the header and schema pages are read before + // any query runs, and that shows up in every later session total. + await this.#account(worker, `open ${this.url}`, performance.now() - opened); return worker; } catch (err) { if (!this.#closed) { @@ -70,21 +83,55 @@ export class RemoteDatabase { } } - /** Run a query and return its rows as objects. */ - async query(sql: string, params: unknown[] = []): Promise { + /** + * Run a query and return its rows as objects. + * + * `label` names the query in the console trace — the only way to see what a + * search actually costs, since the byte count depends on how much the read + * heads prefetched, not just on the pages the plan needed. + */ + async query(sql: string, params: unknown[] = [], label?: string): Promise { const worker = this.#worker ?? (await this.#opening); // Comlink erases the generic when it proxies the method across the worker // boundary, so the row type is asserted here rather than inferred. const run = worker.db.query as unknown as (sql: string, ...params: unknown[]) => Promise; + const started = performance.now(); try { return await run(sql, ...params); } finally { - // Comlink proxies the property, so this is a round trip; worth it because - // the number is the only honest feedback about what a query cost. - this.bytesRead = await worker.worker.bytesRead; + await this.#account(worker, label ?? firstLine(sql), performance.now() - started); } } + /** + * Read the cumulative counters and report the delta. + * + * getStats() rather than the worker's `bytesRead`: that one is the budget + * counter and resets itself to zero when a query trips the ceiling, so it + * would under-report exactly when the number matters most. + */ + async #account(worker: WorkerHttpvfs, label: string, ms: number) { + const read = worker.worker.getStats as unknown as () => Promise; + const stats = await read().catch(() => null); + if (!stats) return; + + const cost: QueryCost = { + requests: stats.totalRequests - this.#seen.requests, + bytes: stats.totalFetchedBytes - this.#seen.bytes, + ms, + }; + this.#seen = { requests: stats.totalRequests, bytes: stats.totalFetchedBytes }; + this.requests = stats.totalRequests; + this.bytesRead = stats.totalFetchedBytes; + this.lastCost = cost; + + console.info( + `[httpvfs] ${label} — ${cost.requests} request(s), ${formatBytes(cost.bytes)}, ${ms.toFixed(0)} ms` + + ` · session ${stats.totalRequests} request(s), ${formatBytes(stats.totalFetchedBytes)}` + + ` of ${formatBytes(stats.totalBytes)}`, + ); + } + /** * Drop this database. createDbWorker owns the Worker and exposes no handle to * it, so the thread outlives this call; a page creates at most one per @@ -106,3 +153,13 @@ export function isBudgetError(err: unknown): boolean { function message(err: unknown): string { return err instanceof Error ? err.message : String(err); } + +/** Enough of a query to recognise it in the console. */ +function firstLine(sql: string): string { + const line = sql.trim().split("\n")[0]; + return line.length > 70 ? `${line.slice(0, 70)}…` : line; +} + +export function formatBytes(n: number): string { + return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`; +} From b38965dc4b7567586eec2837a00761d7c717e327 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 14 Aug 2026 13:49:52 +0700 Subject: [PATCH 4/5] refactor(web): drop TypeScript for plain JavaScript MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every .ts becomes .js, every lang="ts" becomes lang-less, and the type declarations go with them: types.ts held nothing but types, so it is deleted outright. Tooling follows. typescript, svelte-check, typescript-eslint and @types/sql.js are uninstalled; tsconfig.json becomes jsconfig.json, which still extends the generated SvelteKit config so $lib and $app resolve in an editor; `npm run lint` is now ESLint alone, and CI's comment about it covering the type check goes too. What this gives up, stated plainly: a mistyped column name like row.nguvan used to fail the build and now renders blank, and the datasets.json-to-CONTENT cross-check is back to throwing at module load rather than at compile time. The runtime guard for the latter is still there and still throws loudly. Two mechanical notes. The svelte/no-navigation-without-resolve rule started flagging the footer's source link, which points at an off-site article — without type information the rule can no longer tell an external URL from a route, so that one line carries a disable comment. And Vitest's include pattern had to follow the tests to .js. Lint, 25 tests and the build all pass. --- .github/workflows/deploy-pages.yml | 4 +- CLAUDE.md | 7 +- README.md | 4 +- docs/data-pipeline.md | 2 +- docs/system-architecture.md | 8 +- parser/README.md | 2 +- web/eslint.config.js | 17 +- web/{tsconfig.json => jsconfig.json} | 9 +- web/package-lock.json | 473 ------------------ web/package.json | 6 +- ...dmission-blocks.ts => admission-blocks.js} | 25 +- ...locks.test.ts => admission-blocks.test.js} | 3 +- web/src/lib/components/custom-query.svelte | 25 +- web/src/lib/components/score-table.svelte | 7 +- web/src/lib/components/search-form.svelte | 22 +- web/src/lib/components/student-detail.svelte | 15 +- web/src/lib/{datasets.ts => datasets.js} | 13 +- web/src/lib/{query-mode.ts => query-mode.js} | 14 +- ...{query-mode.test.ts => query-mode.test.js} | 0 web/src/lib/{search.ts => search.js} | 20 +- .../lib/{sql-presets.ts => sql-presets.js} | 6 +- .../{sqlite.svelte.ts => sqlite.svelte.js} | 46 +- web/src/lib/{subjects.ts => subjects.js} | 8 +- web/src/lib/{to-ascii.ts => to-ascii.js} | 4 +- .../{to-ascii.test.ts => to-ascii.test.js} | 2 +- web/src/lib/types.ts | 56 --- web/src/routes/{+layout.ts => +layout.js} | 0 web/src/routes/+layout.svelte | 2 +- web/src/routes/+page.svelte | 5 +- .../routes/[dataset]/{+page.ts => +page.js} | 5 +- web/src/routes/[dataset]/+page.svelte | 21 +- web/{vite.config.ts => vite.config.js} | 2 +- 32 files changed, 121 insertions(+), 712 deletions(-) rename web/{tsconfig.json => jsconfig.json} (63%) rename web/src/lib/{admission-blocks.ts => admission-blocks.js} (88%) rename web/src/lib/{admission-blocks.test.ts => admission-blocks.test.js} (96%) rename web/src/lib/{datasets.ts => datasets.js} (87%) rename web/src/lib/{query-mode.ts => query-mode.js} (81%) rename web/src/lib/{query-mode.test.ts => query-mode.test.js} (100%) rename web/src/lib/{search.ts => search.js} (82%) rename web/src/lib/{sql-presets.ts => sql-presets.js} (98%) rename web/src/lib/{sqlite.svelte.ts => sqlite.svelte.js} (79%) rename web/src/lib/{subjects.ts => subjects.js} (84%) rename web/src/lib/{to-ascii.ts => to-ascii.js} (89%) rename web/src/lib/{to-ascii.test.ts => to-ascii.test.js} (97%) delete mode 100644 web/src/lib/types.ts rename web/src/routes/{+layout.ts => +layout.js} (100%) rename web/src/routes/[dataset]/{+page.ts => +page.js} (71%) rename web/{vite.config.ts => vite.config.js} (90%) diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 3b8263d..502a273 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -66,8 +66,8 @@ jobs: - name: Test assembler run: go -C assembler test ./... - # Lint covers ESLint and svelte-check; the tests cover the framework-free - # modules, including the ASCII fold that has to match the Go parser. + # The tests cover the framework-free modules, including the ASCII fold + # that has to match the Go parser. - name: Test web working-directory: web run: npm test diff --git a/CLAUDE.md b/CLAUDE.md index a6b5962..e640750 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,8 @@ Add the web checks when frontend files changed: (cd web && npm test && npm run lint && npm run build) ``` -`npm run lint` is ESLint plus `svelte-check`, so it is the type check too. +`npm run lint` is ESLint. The web app is plain JavaScript — no TypeScript, no +type-check step. **Do not run the crawler suite as part of routine verification.** The crawler is not part of the build — `data//` is committed and a crawl only refreshes it @@ -44,8 +45,8 @@ hashes every real input file. That is the point of it; do not skip it. regenerated.** A mismatch is a reader bug until proven otherwise, never a cue to refresh the file. `parser/cmd/dumpcells` narrows a failure to the cell. - **`ToAscii` filters the literal range U+0300..U+036F, not `unicode.Mn`.** It - must match `toAscii` in `web/src/lib/to-ascii.ts`, or accent-insensitive search - silently misses rows. `to-ascii.test.ts` pins the pairs both sides must agree + must match `toAscii` in `web/src/lib/to-ascii.js`, or accent-insensitive search + silently misses rows. `to-ascii.test.js` pins the pairs both sides must agree on. - **The 2016 files use four different layouts, and detection is per sheet.** Two of them publish scores in one column per subject instead of a `DIEM_THI` diff --git a/README.md b/README.md index f578951..1fcbda7 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Each stage runs on its own and hands its output to the next through the stores. `datasets.json` is the contract between them. It is JSON because Go and the web app both read it and neither needs a dependency to do so; presentation stays in -`web/src/lib/datasets.ts`, keyed by id, which fails loudly if the two disagree. +`web/src/lib/datasets.js`, keyed by id, which fails loudly if the two disagree. The dataset id is one identifier end to end: @@ -86,7 +86,7 @@ Pushing to `main` runs the same steps in 2. Add `parser/configs/.yml` — sheet mode, column indices, validation guards. No SQL; the schema is canonical. 3. Add an entry to `datasets.json` with its expected row count and size -4. Add the matching presentation to `CONTENT` in `web/src/lib/datasets.ts` +4. Add the matching presentation to `CONTENT` in `web/src/lib/datasets.js` Everything else follows: the assembler, the router and the hub all read the registry, and the UI adapts to whichever columns the dataset fills. Steps 3 and 4 diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 1bb9567..518df28 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -183,7 +183,7 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what The databases are written with 1 KiB pages (`PRAGMA page_size` in `parser/internal/writer/writer.go`) because the browser fetches them a page per -HTTP request. `CHUNK_BYTES` in `web/src/lib/sqlite.svelte.ts` must match. +HTTP request. `CHUNK_BYTES` in `web/src/lib/sqlite.svelte.js` must match. | id | Source rows | Skipped | DB rows | | --- | --- | --- | --- | diff --git a/docs/system-architecture.md b/docs/system-architecture.md index 850804a..fd6d5e3 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -46,7 +46,7 @@ because the assembler is a Go program and the web app is not, and JSON is the only format both parse without a dependency. Presentation — titles, labels, search examples, SQL presets — stays in -`web/src/lib/datasets.ts`, keyed by id. That file cross-checks the two: a registry +`web/src/lib/datasets.js`, keyed by id. That file cross-checks the two: a registry entry with no content, or content for a dataset that was never built, throws at module load rather than rendering a page with no title or a link to a database that does not exist. @@ -125,13 +125,13 @@ No component contains a per-dataset conditional. Two mechanisms do the work: self-exclude wherever those languages were not sat. Anything genuinely per-dataset — title, source, database size, search examples, -SQL presets — lives in `web/src/lib/datasets.ts`. +SQL presets — lives in `web/src/lib/datasets.js`. ## Exam ID formats -`web/src/lib/query-mode.ts` decides whether a query is an exam ID or a name, and +`web/src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is shared by the dataset page and `search-form.svelte` (they previously held -separate copies and had drifted apart on exactly this rule). `query-mode.test.ts` +separate copies and had drifted apart on exactly this rule). `query-mode.test.js` covers every form in the table below. | Form | Example | Where | diff --git a/parser/README.md b/parser/README.md index 23f57e1..3fea335 100644 --- a/parser/README.md +++ b/parser/README.md @@ -41,7 +41,7 @@ reader independently verifiable against a hash oracle. - `ToAscii` strips combining marks in the literal range U+0300–U+036F rather than by Unicode category. That covers every Vietnamese diacritic and must - stay identical to `toAscii` in `web/src/lib/to-ascii.ts`, or accent-insensitive + stay identical to `toAscii` in `web/src/lib/to-ascii.js`, or accent-insensitive search misses rows. - Gender is normalised to `Nam`/`Nữ`; the Cần Thơ files write `0`/`1` instead and are translated. Anything else becomes NULL. diff --git a/web/eslint.config.js b/web/eslint.config.js index 2175062..6378a41 100644 --- a/web/eslint.config.js +++ b/web/eslint.config.js @@ -1,12 +1,10 @@ import js from "@eslint/js"; import svelte from "eslint-plugin-svelte"; import globals from "globals"; -import ts from "typescript-eslint"; import svelteConfig from "./svelte.config.js"; -export default ts.config( +export default [ js.configs.recommended, - ...ts.configs.recommended, ...svelte.configs.recommended, { languageOptions: { @@ -15,16 +13,11 @@ export default ts.config( }, { // Svelte files and rune modules are parsed by svelte-eslint-parser, which - // needs the TypeScript parser handed to it for `lang="ts"` blocks. - files: ["**/*.svelte", "**/*.svelte.ts", "**/*.svelte.js"], + // needs the project's svelte.config.js to resolve aliases and runes. + files: ["**/*.svelte", "**/*.svelte.js"], languageOptions: { - parserOptions: { - parser: ts.parser, - projectService: true, - extraFileExtensions: [".svelte"], - svelteConfig, - }, + parserOptions: { svelteConfig }, }, }, { ignores: ["dist/", ".svelte-kit/", "node_modules/"] }, -); +]; diff --git a/web/tsconfig.json b/web/jsconfig.json similarity index 63% rename from web/tsconfig.json rename to web/jsconfig.json index 4344710..21df54b 100644 --- a/web/tsconfig.json +++ b/web/jsconfig.json @@ -1,14 +1,11 @@ { "extends": "./.svelte-kit/tsconfig.json", "compilerOptions": { - "allowJs": true, - "checkJs": true, + "moduleResolution": "bundler", + "resolveJsonModule": true, "esModuleInterop": true, "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, "skipLibCheck": true, - "sourceMap": true, - "strict": true, - "moduleResolution": "bundler" + "sourceMap": true } } diff --git a/web/package-lock.json b/web/package-lock.json index 5ed552d..6f55121 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -16,15 +16,11 @@ "@sveltejs/kit": "^2.70.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.3.3", - "@types/sql.js": "^1.4.11", "eslint": "^9.39.4", "eslint-plugin-svelte": "^3.14.0", "globals": "^17.4.0", "svelte": "^5.56.9", - "svelte-check": "^4.4.3", "tailwindcss": "^4.3.3", - "typescript": "^5.9.3", - "typescript-eslint": "^8.48.2", "vite": "^7.1.14", "vitest": "^4.1.10" } @@ -1187,16 +1183,6 @@ } } }, - "node_modules/@sveltejs/load-config": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.3.tgz", - "integrity": "sha512-VT3qmUb8pRV2QrZjd8iAmtg8lf4W0TIjZbvXtz5MKei/q96teWZgGJyyidJzOjzZzvdq616eSRVeMYIQChUTAQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 18.0.0" - } - }, "node_modules/@sveltejs/vite-plugin-svelte": { "version": "6.2.4", "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-6.2.4.tgz", @@ -1818,13 +1804,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/emscripten": { - "version": "1.41.5", - "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", - "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/estree": { "version": "1.0.9", "dev": true, @@ -1835,27 +1814,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" - } - }, - "node_modules/@types/sql.js": { - "version": "1.4.11", - "resolved": "https://registry.npmjs.org/@types/sql.js/-/sql.js-1.4.11.tgz", - "integrity": "sha512-QXIx38p2ZThJaK9vP5ZdqdlRe1FG9I8SmCZOS7FHfB/2qPAjZwkL7/vlfPg6N/oWHuuOaGg/P/IRwfP2W0kWVQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/emscripten": "*", - "@types/node": "*" - } - }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -1863,301 +1821,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.67.0.tgz", - "integrity": "sha512-Un7Heoyj65NREbKAyIrFxeM143NZpExWmy1Nep4DLeQOeLlTeumPjoNKnBrU5D5moWXbPJgRa5Uwcdu0faVNGQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/type-utils": "8.67.0", - "@typescript-eslint/utils": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "ignore": "^7.0.5", - "natural-compare": "^1.4.0", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^8.67.0", - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", - "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.67.0.tgz", - "integrity": "sha512-fUBfTuuEulWqX6V8+O3PtScV01tzYYRUDTAirHFKoRAt7nOzoGiPt0M/bB47wWNy0coOOcgEwAMUtBpykMxl6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/project-service": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.67.0.tgz", - "integrity": "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.67.0", - "@typescript-eslint/types": "^8.67.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.67.0.tgz", - "integrity": "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.67.0.tgz", - "integrity": "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.67.0.tgz", - "integrity": "sha512-aVWDXbRmdXO9siTfX4ditQI1T9+zVcNazT48EJCD0v40/9RIFoUgZ05CmGEq9H2gixRpjUn/iplwvlcvutJW/Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/utils": "8.67.0", - "debug": "^4.4.3", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/types": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.67.0.tgz", - "integrity": "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.67.0.tgz", - "integrity": "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/project-service": "8.67.0", - "@typescript-eslint/tsconfig-utils": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/visitor-keys": "8.67.0", - "debug": "^4.4.3", - "minimatch": "^10.2.2", - "semver": "^7.7.3", - "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.9", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", - "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.8" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@typescript-eslint/utils": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.67.0.tgz", - "integrity": "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.67.0", - "@typescript-eslint/types": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.67.0.tgz", - "integrity": "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.67.0", - "eslint-visitor-keys": "^5.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/@vitest/expect": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", @@ -2401,22 +2064,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -3189,16 +2836,6 @@ "node": "*" } }, - "node_modules/mri": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", - "integrity": "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -3495,20 +3132,6 @@ "node": ">=6" } }, - "node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "dev": true, @@ -3563,19 +3186,6 @@ "fsevents": "~2.3.2" } }, - "node_modules/sade": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", - "integrity": "sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mri": "^1.1.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/set-cookie-parser": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.2.tgz", @@ -3705,31 +3315,6 @@ "node": ">=18" } }, - "node_modules/svelte-check": { - "version": "4.7.6", - "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.6.tgz", - "integrity": "sha512-t2scM//ZuVbSY/T2w6FSBw1v9s2NEmh/g+sy1lqtosW5ylBV5AF4wFb1Ts9Kf3MbfPDUDJDZ9L436YT0SPTdvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "@sveltejs/load-config": "^0.2.3", - "chokidar": "^4.0.1", - "fdir": "^6.2.0", - "picocolors": "^1.0.0", - "sade": "^1.7.4" - }, - "bin": { - "svelte-check": "bin/svelte-check" - }, - "engines": { - "node": ">= 18.0.0" - }, - "peerDependencies": { - "svelte": "^4.0.0 || ^5.0.0-next.0", - "typescript": "^5.0.0 || ^6.0.0" - } - }, "node_modules/svelte-eslint-parser": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-1.8.0.tgz", @@ -3847,19 +3432,6 @@ "node": ">=6" } }, - "node_modules/ts-api-utils": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", - "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.12" - }, - "peerDependencies": { - "typescript": ">=4.8.4" - } - }, "node_modules/type-check": { "version": "0.4.0", "dev": true, @@ -3871,51 +3443,6 @@ "node": ">= 0.8.0" } }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/typescript-eslint": { - "version": "8.67.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.67.0.tgz", - "integrity": "sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/eslint-plugin": "8.67.0", - "@typescript-eslint/parser": "8.67.0", - "@typescript-eslint/typescript-estree": "8.67.0", - "@typescript-eslint/utils": "8.67.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", - "dev": true, - "license": "MIT" - }, "node_modules/uri-js": { "version": "4.4.1", "dev": true, diff --git a/web/package.json b/web/package.json index fbf0a85..646acc1 100644 --- a/web/package.json +++ b/web/package.json @@ -9,7 +9,7 @@ "build": "vite build", "preview": "vite preview", "test": "vitest run", - "lint": "eslint . && svelte-check --tsconfig ./tsconfig.json" + "lint": "eslint ." }, "dependencies": { "sql.js-httpvfs": "^0.8.12" @@ -20,15 +20,11 @@ "@sveltejs/kit": "^2.70.2", "@sveltejs/vite-plugin-svelte": "^6.2.1", "@tailwindcss/vite": "^4.3.3", - "@types/sql.js": "^1.4.11", "eslint": "^9.39.4", "eslint-plugin-svelte": "^3.14.0", "globals": "^17.4.0", "svelte": "^5.56.9", - "svelte-check": "^4.4.3", "tailwindcss": "^4.3.3", - "typescript": "^5.9.3", - "typescript-eslint": "^8.48.2", "vite": "^7.1.14", "vitest": "^4.1.10" } diff --git a/web/src/lib/admission-blocks.ts b/web/src/lib/admission-blocks.js similarity index 88% rename from web/src/lib/admission-blocks.ts rename to web/src/lib/admission-blocks.js index 98f5142..b99c600 100644 --- a/web/src/lib/admission-blocks.ts +++ b/web/src/lib/admission-blocks.js @@ -1,15 +1,3 @@ -import type { Student, SubjectKey } from "./types"; - -export type AdmissionBlock = { code: string; subjects: SubjectKey[]; label: string }; -export type ComputedBlock = { - code: string; - label: string; - total: number; - parts: { key: SubjectKey; score: number }[]; -}; -export type TierKey = "common" | "uncommon" | "rare" | "epic" | "legendary" | "prismatic"; -export type Tier = { key: TierKey; symbol: string; label: string }; - // Vietnamese university admission-block ("khối thi") definitions, per Circular // 03/2017/TT-BGDĐT. Each block is the sum of exactly 3 subject scores. // @@ -17,7 +5,7 @@ export type Tier = { key: TierKey; symbol: string; label: string }; // skips any block with a missing subject score, so blocks needing GDCD // self-exclude on 2016 rows, and blocks needing a language the candidate did // not sit self-exclude row by row. -export const ADMISSION_BLOCKS: AdmissionBlock[] = [ +export const ADMISSION_BLOCKS = [ { code: "A00", subjects: ["toan", "vat_ly", "hoa_hoc"], label: "Toán + Lý + Hóa" }, { code: "A01", subjects: ["toan", "vat_ly", "tieng_anh"], label: "Toán + Lý + Anh" }, { code: "A02", subjects: ["toan", "vat_ly", "sinh_hoc"], label: "Toán + Lý + Sinh" }, @@ -76,14 +64,13 @@ export const ADMISSION_BLOCKS: AdmissionBlock[] = [ // Returns { code, label, total, parts:[{key,score}] } for every block where // the student has scores for all three subjects, sorted by total desc. -export function computeBlocks(student: Student): ComputedBlock[] { - const out: ComputedBlock[] = []; +export function computeBlocks(student) { + const out = []; for (const b of ADMISSION_BLOCKS) { const parts = b.subjects.map((k) => ({ key: k, score: student[k] })); if (parts.some((p) => p.score === null || p.score === undefined)) continue; - const whole = parts as { key: SubjectKey; score: number }[]; - const total = whole.reduce((s, p) => s + p.score, 0); - out.push({ code: b.code, label: b.label, total, parts: whole }); + const total = parts.reduce((s, p) => s + p.score, 0); + out.push({ code: b.code, label: b.label, total, parts }); } return out.sort((a, b) => b.total - a.total); } @@ -93,7 +80,7 @@ export function computeBlocks(student: Student): ComputedBlock[] { // whatever the other scores are, so that boundary is inclusive while the rest // are exclusive upper bounds. Every tier carries a unicode symbol as well as a // color, so the meaning is never conveyed by color alone. -export function scoreTier(score: number | null | undefined): Tier | null { +export function scoreTier(score) { if (score === null || score === undefined) return null; if (score <= 1) return { key: "common", symbol: "·", label: "Điểm liệt" }; if (score < 5) return { key: "uncommon", symbol: "○", label: "Chưa đạt" }; diff --git a/web/src/lib/admission-blocks.test.ts b/web/src/lib/admission-blocks.test.js similarity index 96% rename from web/src/lib/admission-blocks.test.ts rename to web/src/lib/admission-blocks.test.js index d78f189..8239e6e 100644 --- a/web/src/lib/admission-blocks.test.ts +++ b/web/src/lib/admission-blocks.test.js @@ -1,9 +1,8 @@ import { describe, expect, it } from "vitest"; import { ADMISSION_BLOCKS, computeBlocks, scoreTier } from "./admission-blocks"; -import type { Student } from "./types"; /** A row with every column NULL, so a test only states the scores it needs. */ -function student(scores: Partial): Student { +function student(scores) { return { so_bao_danh: "49008235", ho_ten: "Nguyễn Văn A", diff --git a/web/src/lib/components/custom-query.svelte b/web/src/lib/components/custom-query.svelte index d18df0b..dd0734c 100644 --- a/web/src/lib/components/custom-query.svelte +++ b/web/src/lib/components/custom-query.svelte @@ -1,29 +1,24 @@ - diff --git a/web/src/routes/[dataset]/+page.ts b/web/src/routes/[dataset]/+page.js similarity index 71% rename from web/src/routes/[dataset]/+page.ts rename to web/src/routes/[dataset]/+page.js index 055bd77..b9d5a57 100644 --- a/web/src/routes/[dataset]/+page.ts +++ b/web/src/routes/[dataset]/+page.js @@ -1,15 +1,14 @@ import { error } from "@sveltejs/kit"; import { DATASETS, datasetById } from "$lib/datasets"; -import type { EntryGenerator, PageLoad } from "./$types"; /** * One prerendered page per registered dataset. The registry is the repository * root's datasets.json, so a dataset the assembler builds is a page that exists, * and one it does not is a 404 at build time rather than a blank page. */ -export const entries: EntryGenerator = () => DATASETS.map((d) => ({ dataset: d.id })); +export const entries = () => DATASETS.map((d) => ({ dataset: d.id })); -export const load: PageLoad = ({ params }) => { +export const load = ({ params }) => { const dataset = datasetById(params.dataset); if (!dataset) throw error(404, `Unknown dataset: ${params.dataset}`); return { dataset }; diff --git a/web/src/routes/[dataset]/+page.svelte b/web/src/routes/[dataset]/+page.svelte index ac18283..e77d48e 100644 --- a/web/src/routes/[dataset]/+page.svelte +++ b/web/src/routes/[dataset]/+page.svelte @@ -1,4 +1,4 @@ -