mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-05 12:14:24 +00:00
The browser downloaded 45 MB of gzipped SQLite before it could answer anything. Now sql.js-httpvfs asks for the pages a query touches and the databases ship uncompressed as <id>.sqlite3 — a byte range of a gzip stream is not a byte range of a database. That only works if every query the site issues is index-driven, and measured against the real 2016 file, most were not: so_bao_danh = ? SEARCH via PK ~20 KB ho_ten_ascii LIKE '%x%' SCAN 127 MB ho_ten_ascii LIKE 'x%' SCAN 127 MB COUNT(*) covering index scan 20 MB ORDER BY toan DESC LIMIT 10 SCAN + temp b-tree 127 MB Prefix LIKE scans because SQLite's LIKE optimisation needs a NOCASE index; a range comparison does use the index. So the schema changed to suit the access pattern rather than the search changing to suit the schema. name_word holds one row per word of each name, WITHOUT ROWID so the table is the index, carrying ho_ten_ascii so a multi-word query is resolved inside a single b-tree. name_word_freq says which word of a query is rarest — the vocabulary is 4,397 words across 2.87M entries, so "buu loc" seeks on 287 entries rather than walking the 300,000 that "thi" would. Searching by any word of a name survives, at a few hundred KB a query. idx_ho_ten and idx_ho_ten_ascii are gone: no plan could use either. Partial indexes on toan, khtn and khxh cost 12 MB and keep the SQL presets off a full scan. The footer's candidate count now comes from datasets.json instead of COUNT(*). 2016 grows 223.5 MB to 288.6 MB, 2017 162.7 MB to 237.7 MB, and the site is 528 MB against the 1 GB GitHub Pages limit. Row counts are unchanged. The SQL tab is the one place a user can still write a query that reads the whole table, so it asks before it opens, runs under a byte budget that stops a runaway query, and shows what each query actually fetched. Verified: row counts through the assembler guards, every app query index-driven under EXPLAIN QUERY PLAN, and GitHub Pages returning 206 with a correct Content-Range. Not verified in a browser — this machine has none — and the library refuses to open a file the host compresses, so the deployed response headers need a look.
163 lines
5.4 KiB
Go
163 lines
5.4 KiB
Go
package site
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/tiennm99/thptqg/assembler/internal/registry"
|
|
)
|
|
|
|
var datasets = []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
|
|
|
|
// fakeBuild stands in for a web build: the prerendered hub and one page per
|
|
// dataset, an asset, and whatever databases the caller wants staged.
|
|
func fakeBuild(t *testing.T, dbs ...string) Paths {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
dist := filepath.Join(root, "web", "dist")
|
|
write(t, filepath.Join(dist, "index.html"), "<html>hub</html>")
|
|
for _, d := range datasets {
|
|
write(t, filepath.Join(dist, d.ID, "index.html"), "<html>"+d.ID+"</html>")
|
|
}
|
|
write(t, filepath.Join(dist, "_app", "immutable", "entry.js"), "console.log(1)")
|
|
for _, name := range dbs {
|
|
write(t, filepath.Join(dist, "db", name), "sqlite-bytes")
|
|
}
|
|
return Paths{Web: filepath.Join(root, "web"), Dist: dist, Site: filepath.Join(root, "_site")}
|
|
}
|
|
|
|
func write(t *testing.T, path, body string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
|
if err := Assemble(p, datasets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{
|
|
"index.html",
|
|
"404.html",
|
|
filepath.Join("2016", "index.html"),
|
|
filepath.Join("2017", "index.html"),
|
|
filepath.Join("_app", "immutable", "entry.js"),
|
|
filepath.Join("db", "2016.sqlite3"),
|
|
} {
|
|
if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil {
|
|
t.Errorf("missing from the artifact: %s", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMissingDatasetPageFailsTheBuild: the pages come from the web build's
|
|
// entry generator, which reads the same registry this does. If the two fall out
|
|
// of step, that dataset's URL 404s — so the build stops instead.
|
|
func TestMissingDatasetPageFailsTheBuild(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
|
if err := os.RemoveAll(filepath.Join(p.Dist, "2017")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := Assemble(p, datasets)
|
|
if err == nil {
|
|
t.Fatal("expected an error when a dataset page was not prerendered")
|
|
}
|
|
if !strings.Contains(err.Error(), "2017") {
|
|
t.Errorf("the error should name the missing page, got: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestMissingDatabaseFailsTheBuild is the guard that closes the widest hole.
|
|
//
|
|
// Without it the site assembles happily with an empty db/ directory: every page
|
|
// renders, every query 404s, and CI stays green. The row-count and size guards
|
|
// cannot catch this — they only run when a database was built at all.
|
|
func TestMissingDatabaseFailsTheBuild(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3") // 2017 never built
|
|
err := Assemble(p, datasets)
|
|
if err == nil {
|
|
t.Fatal("expected an error when a database is missing")
|
|
}
|
|
if !strings.Contains(err.Error(), "2017.sqlite3") {
|
|
t.Errorf("the error should name the missing database, got: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is
|
|
// not a database.
|
|
func TestEmptyDatabaseFailsTheBuild(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
|
write(t, filepath.Join(p.Dist, "db", "2017.sqlite3"), "")
|
|
if err := Assemble(p, datasets); err == nil {
|
|
t.Fatal("expected an error for a zero-byte database")
|
|
}
|
|
}
|
|
|
|
// TestStrayArtifactFailsTheBuild: a journal means an interrupted run, a .db
|
|
// means the old naming, a .gz means a database the site could not read a range
|
|
// of — and each is 100+ MB.
|
|
func TestStrayArtifactFailsTheBuild(t *testing.T) {
|
|
for _, name := range []string{
|
|
"2016.db", "2016.sqlite3-journal", "2016.sqlite3-wal", "2016.sqlite3-shm", "2016.sqlite3.gz",
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
|
write(t, filepath.Join(p.Dist, "db", name), "raw sqlite")
|
|
err := Assemble(p, datasets)
|
|
if err == nil {
|
|
t.Fatalf("expected an error for %s", name)
|
|
}
|
|
if !strings.Contains(err.Error(), "stray") {
|
|
t.Errorf("unexpected error: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestPublishedDatabaseIsNotMistakenForStray: the pattern must pass the one
|
|
// file the site is built to serve. Getting this wrong would fail every build.
|
|
func TestPublishedDatabaseIsNotMistakenForStray(t *testing.T) {
|
|
if strayArtifact.MatchString("2016.sqlite3") {
|
|
t.Error("the published database must not be treated as a stray artifact")
|
|
}
|
|
for _, name := range []string{
|
|
"2016.db", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm", "x.sqlite3.gz", "x.db.gz",
|
|
} {
|
|
if !strayArtifact.MatchString(name) {
|
|
t.Errorf("%s should be rejected", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAssembleRejectsAMissingBuild(t *testing.T) {
|
|
root := t.TempDir()
|
|
p := Paths{Web: root, Dist: filepath.Join(root, "dist"), Site: filepath.Join(root, "_site")}
|
|
if err := Assemble(p, datasets); err == nil {
|
|
t.Fatal("expected an error when there is no web build")
|
|
}
|
|
}
|
|
|
|
// TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a
|
|
// previous run's leftovers cannot survive into the artifact.
|
|
func TestAssembleIsIdempotent(t *testing.T) {
|
|
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
|
if err := Assemble(p, datasets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stale := filepath.Join(p.Site, "2015", "index.html")
|
|
write(t, stale, "old dataset")
|
|
if err := Assemble(p, datasets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(stale); !os.IsNotExist(err) {
|
|
t.Error("a directory from a previous run survived into the new artifact")
|
|
}
|
|
}
|