Files
tiennm99 4cb0a4f340 feat: download the database instead of reading it over HTTP
Reading the file where it lay never worked well enough. Two costs were
structural rather than bugs: reads are serial, because the worker uses
synchronous XHR, so a name search touching 390 pages waited 17 seconds
to move 608 KB — roughly one request per result row, which no page size
removes — and the first visitor after each deploy waited ~26 seconds for
the CDN to fill its cache with a 288 MB object.

The browser now downloads the whole database once and queries it in
memory with sql.js. A dataset page is gated behind that: the gate states
what it will cost, in transfer and in memory, and offers only the
download, because there is nothing to show without it.

Dropping the structures that existed to make range-request queries
index-driven halved the file. name_word carried one row per word of
every name, about 3.5 million of them, and with the partial score
indexes it was more than half of what every visitor would now download.
Measured on rebuilt databases: 2016 went 288.6 -> 142.5 MB (31 MB
gzipped on the wire), 2017 237.7 -> 119.3 MB, both with row counts and
audits unchanged. Queries on the result: an exam number is immediate, a
name scans all 877,460 rows in about 240 ms.

Alternatives were measured before choosing this. sqlite-wasm-http sizes
files from a HEAD Content-Length with no override, so on a host that
gzips it silently uses the compressed size. DuckDB-WASM ships 32-37 MB
of WebAssembly before its Parquet extension, more than this whole
download. Static pre-generated shards are the most robust option but
cannot answer arbitrary SQL, and cannot stop early the way LIMIT does.

The published name loses its chunk index, the byte budgets and the SQL
consent modal go with the range reads that made them necessary, and the
docs no longer describe a design the site does not use.
2026-08-14 17:43:10 +07:00

154 lines
4.4 KiB
Go

package schema
import (
"strings"
"testing"
)
// These tests stop the DDL, the INSERT column list and the field-order constants
// from drifting apart — a drift that silently lands values in the wrong columns.
func TestInsertMatchesFieldOrder(t *testing.T) {
if ParamCount != 22 {
t.Errorf("ParamCount = %d, want 22", ParamCount)
}
if got := strings.Count(InsertSQL, "?"); got != ParamCount {
t.Errorf("INSERT placeholders = %d, want %d", got, ParamCount)
}
open := strings.Index(InsertSQL, "(")
closeIdx := strings.Index(InsertSQL, ")")
if open < 0 || closeIdx < 0 {
t.Fatal("INSERT must contain a column list")
}
var listed []string
for _, c := range strings.Split(InsertSQL[open+1:closeIdx], ",") {
if c = strings.TrimSpace(c); c != "" {
listed = append(listed, c)
}
}
want := append(append([]string{}, IdentityFields...), ScoreFields...)
if len(listed) != len(want) {
t.Fatalf("INSERT lists %d columns, want %d", len(listed), len(want))
}
for i := range want {
if listed[i] != want[i] {
t.Errorf("column %d: INSERT has %q, field order has %q", i, listed[i], want[i])
}
}
}
func TestScorePatternsCoverScoreFields(t *testing.T) {
if len(ScorePatterns) != len(ScoreFields) {
t.Fatalf("%d patterns for %d score columns", len(ScorePatterns), len(ScoreFields))
}
inFields := make(map[string]bool, len(ScoreFields))
for _, f := range ScoreFields {
inFields[f] = true
}
for field := range ScorePatterns {
if !inFields[field] {
t.Errorf("pattern %q has no column", field)
}
}
for _, field := range ScoreFields {
if _, ok := ScorePatterns[field]; !ok {
t.Errorf("column %q has no pattern", field)
}
}
}
func TestDDLColumnsMatchInsert(t *testing.T) {
for _, field := range append(append([]string{}, IdentityFields...), ScoreFields...) {
if !strings.Contains(DDL, field) {
t.Errorf("DDL missing column %q", field)
}
}
}
// TestScorePatternsCompile: compilation happens in the package initialiser, so
// reaching this point already proves it; the explicit checks guard against an
// empty or partial table.
func TestScorePatternsCompile(t *testing.T) {
for field, re := range ScorePatterns {
if re == nil {
t.Errorf("pattern %q is nil", field)
}
}
}
// TestDDLIsFrozen compares DDL against an independent copy of the exact text,
// down to the byte. It catches column, type and index changes that every
// row-level check would still pass — a database can be structurally different
// and look fine one row at a time. Update the copy below only when the schema
// change is intended.
func TestDDLIsFrozen(t *testing.T) {
const want = `
CREATE TABLE student (
so_bao_danh TEXT PRIMARY KEY,
ho_ten TEXT NOT NULL,
ho_ten_ascii TEXT NOT NULL,
ngay_sinh TEXT,
ten_cum_thi TEXT,
gioi_tinh TEXT,
toan REAL,
ngu_van REAL,
vat_ly REAL,
hoa_hoc REAL,
sinh_hoc REAL,
khtn REAL,
lich_su REAL,
dia_ly REAL,
gdcd REAL,
khxh REAL,
tieng_anh REAL,
tieng_phap REAL,
tieng_nga REAL,
tieng_duc REAL,
tieng_nhat REAL,
tieng_trung REAL
);
`
if DDL != want {
t.Errorf("DDL changed\n--- got ---\n%s\n--- want ---\n%s", DDL, want)
}
}
// TestNoSecondaryIndexes: the browser downloads this file whole and queries it
// in memory, where a scan of 877,460 rows takes a few hundred milliseconds. An
// index would save that and cost every visitor tens of megabytes of download.
func TestNoSecondaryIndexes(t *testing.T) {
if strings.Contains(DDL, "CREATE INDEX") {
t.Error("DDL creates an index; every one of them is paid for on the network")
}
}
// TestScorePatternsMatchScores exercises each pattern against the shape the
// DIEM_THI cell actually carries, including the wide runs of spaces seen in the
// real corpus.
func TestScorePatternsMatchScores(t *testing.T) {
const cell = "Toán: 8.50 Ngữ văn: 7.00 Tiếng Đức: 9 KHXH: 5.58 "
cases := map[string]string{
"toan": "8.50",
"ngu_van": "7.00",
"tieng_duc": "9",
"khxh": "5.58",
"tieng_nhat": "", // absent from the cell -> no match
}
for field, want := range cases {
re, ok := ScorePatterns[field]
if !ok {
t.Fatalf("no pattern for %q", field)
}
m := re.FindStringSubmatch(cell)
got := ""
if m != nil {
got = m[1]
}
if got != want {
t.Errorf("%s: matched %q, want %q", field, got, want)
}
}
}