mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-08-25 09:24:49 +00:00
tiennm99
dbc23c25c5
The browser downloaded 45 MB of gzipped SQLite before it could answer anything. Now sql.js-httpvfs asks for the pages a query touches and the databases ship uncompressed as <id>.sqlite3 — a byte range of a gzip stream is not a byte range of a database. That only works if every query the site issues is index-driven, and measured against the real 2016 file, most were not: so_bao_danh = ? SEARCH via PK ~20 KB ho_ten_ascii LIKE '%x%' SCAN 127 MB ho_ten_ascii LIKE 'x%' SCAN 127 MB COUNT(*) covering index scan 20 MB ORDER BY toan DESC LIMIT 10 SCAN + temp b-tree 127 MB Prefix LIKE scans because SQLite's LIKE optimisation needs a NOCASE index; a range comparison does use the index. So the schema changed to suit the access pattern rather than the search changing to suit the schema. name_word holds one row per word of each name, WITHOUT ROWID so the table is the index, carrying ho_ten_ascii so a multi-word query is resolved inside a single b-tree. name_word_freq says which word of a query is rarest — the vocabulary is 4,397 words across 2.87M entries, so "buu loc" seeks on 287 entries rather than walking the 300,000 that "thi" would. Searching by any word of a name survives, at a few hundred KB a query. idx_ho_ten and idx_ho_ten_ascii are gone: no plan could use either. Partial indexes on toan, khtn and khxh cost 12 MB and keep the SQL presets off a full scan. The footer's candidate count now comes from datasets.json instead of COUNT(*). 2016 grows 223.5 MB to 288.6 MB, 2017 162.7 MB to 237.7 MB, and the site is 528 MB against the 1 GB GitHub Pages limit. Row counts are unchanged. The SQL tab is the one place a user can still write a query that reads the whole table, so it asks before it opens, runs under a byte budget that stops a runaway query, and shows what each query actually fetched. Verified: row counts through the assembler guards, every app query index-driven under EXPLAIN QUERY PLAN, and GitHub Pages returning 206 with a correct Content-Range. Not verified in a browser — this machine has none — and the library refuses to open a file the host compresses, so the deployed response headers need a look.
67 lines
1.7 KiB
Go
67 lines
1.7 KiB
Go
package databases
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/tiennm99/thptqg/assembler/internal/registry"
|
|
)
|
|
|
|
func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for _, name := range []string{
|
|
"2016.sqlite3", "2017.sqlite3",
|
|
"2017-old.sqlite3", // dropped from the registry
|
|
"2017-old2.sqlite3", // dropped from the registry
|
|
"2016.db-journal", // interrupted run
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
p := Paths{OutDir: dir}
|
|
keep := []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
|
|
if err := Clean(p, keep); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var left []string
|
|
for _, e := range entries {
|
|
left = append(left, e.Name())
|
|
}
|
|
slices.Sort(left)
|
|
|
|
want := []string{"2016.sqlite3", "2017.sqlite3"}
|
|
if !slices.Equal(left, want) {
|
|
t.Errorf("left %v, want %v", left, want)
|
|
}
|
|
}
|
|
|
|
// TestCleanToleratesAnAbsentStagingDirectory: a fresh checkout has never built
|
|
// anything, and that is not an error.
|
|
func TestCleanToleratesAnAbsentStagingDirectory(t *testing.T) {
|
|
p := Paths{OutDir: filepath.Join(t.TempDir(), "never-created")}
|
|
if err := Clean(p, nil); err != nil {
|
|
t.Errorf("Clean on a missing directory should succeed, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDefaultPaths(t *testing.T) {
|
|
p := DefaultPaths("/repo")
|
|
if p.Parser != filepath.Join("/repo", "parser") {
|
|
t.Errorf("Parser = %q", p.Parser)
|
|
}
|
|
// The staging directory must be the one Vite publishes, or the databases
|
|
// never reach the site.
|
|
if p.OutDir != filepath.Join("/repo", ".build", "public", "db") {
|
|
t.Errorf("OutDir = %q", p.OutDir)
|
|
}
|
|
}
|