mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-08-15 09:22:16 +00:00
tiennm99
4cb0a4f340
Reading the file where it lay never worked well enough. Two costs were structural rather than bugs: reads are serial, because the worker uses synchronous XHR, so a name search touching 390 pages waited 17 seconds to move 608 KB — roughly one request per result row, which no page size removes — and the first visitor after each deploy waited ~26 seconds for the CDN to fill its cache with a 288 MB object. The browser now downloads the whole database once and queries it in memory with sql.js. A dataset page is gated behind that: the gate states what it will cost, in transfer and in memory, and offers only the download, because there is nothing to show without it. Dropping the structures that existed to make range-request queries index-driven halved the file. name_word carried one row per word of every name, about 3.5 million of them, and with the partial score indexes it was more than half of what every visitor would now download. Measured on rebuilt databases: 2016 went 288.6 -> 142.5 MB (31 MB gzipped on the wire), 2017 237.7 -> 119.3 MB, both with row counts and audits unchanged. Queries on the result: an exam number is immediate, a name scans all 877,460 rows in about 240 ms. Alternatives were measured before choosing this. sqlite-wasm-http sizes files from a HEAD Content-Length with no override, so on a host that gzips it silently uses the compressed size. DuckDB-WASM ships 32-37 MB of WebAssembly before its Parquet extension, more than this whole download. Static pre-generated shards are the most robust option but cannot answer arbitrary SQL, and cannot stop early the way LIMIT does. The published name loses its chunk index, the byte budgets and the SQL consent modal go with the range reads that made them necessary, and the docs no longer describe a design the site does not use.
67 lines
1.7 KiB
Go
67 lines
1.7 KiB
Go
package databases
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/tiennm99/thptqg/assembler/internal/registry"
|
|
)
|
|
|
|
func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for _, name := range []string{
|
|
"2016.sqlite3", "2017.sqlite3",
|
|
"2017-old.sqlite3", // dropped from the registry
|
|
"2017-old2.sqlite3", // dropped from the registry
|
|
"2016.db-journal", // interrupted run
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
p := Paths{OutDir: dir}
|
|
keep := []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
|
|
if err := Clean(p, keep); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var left []string
|
|
for _, e := range entries {
|
|
left = append(left, e.Name())
|
|
}
|
|
slices.Sort(left)
|
|
|
|
want := []string{"2016.sqlite3", "2017.sqlite3"}
|
|
if !slices.Equal(left, want) {
|
|
t.Errorf("left %v, want %v", left, want)
|
|
}
|
|
}
|
|
|
|
// TestCleanToleratesAnAbsentStagingDirectory: a fresh checkout has never built
|
|
// anything, and that is not an error.
|
|
func TestCleanToleratesAnAbsentStagingDirectory(t *testing.T) {
|
|
p := Paths{OutDir: filepath.Join(t.TempDir(), "never-created")}
|
|
if err := Clean(p, nil); err != nil {
|
|
t.Errorf("Clean on a missing directory should succeed, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDefaultPaths(t *testing.T) {
|
|
p := DefaultPaths("/repo")
|
|
if p.Parser != filepath.Join("/repo", "parser") {
|
|
t.Errorf("Parser = %q", p.Parser)
|
|
}
|
|
// The staging directory must be the one Vite publishes, or the databases
|
|
// never reach the site.
|
|
if p.OutDir != filepath.Join("/repo", ".build", "public", "db") {
|
|
t.Errorf("OutDir = %q", p.OutDir)
|
|
}
|
|
}
|