feat(web): trace what each query costs over the network

The whole design rests on a claim nobody has measured: that a lookup
fetches a few hundred KB rather than the file. Every query now reports
what it actually cost.

  [httpvfs] name "nguyen buu loc" seeking "buu" — 14 request(s), 78 KB,
  212 ms · session 31 request(s), 180 KB of 302.4 MB

The numbers come from the worker's getStats() rather than its bytesRead
counter: bytesRead is the budget accumulator and resets itself to zero
the moment a query trips the ceiling, so it would under-report exactly
when the number matters. getStats() reports per-file totals including
what the read heads prefetched, which is the honest figure — prefetch
overfetch is invisible to a page count.

Opening a database logs too, since the header and schema pages are read
before any query and would otherwise inflate the first search.

The SQL tab shows the same pair on screen: this query, then the session.
This commit is contained in:
2026-08-14 13:43:11 +07:00
parent dbf13d0094
commit d7e2fa2d12
3 changed files with 83 additions and 21 deletions
+8 -8
View File
@@ -1,5 +1,5 @@
<script lang="ts">
import { isBudgetError, type RemoteDatabase } from "$lib/sqlite.svelte";
import { formatBytes, isBudgetError, type RemoteDatabase } from "$lib/sqlite.svelte";
import type { PresetGroup } from "$lib/types";
const MAX_ROWS = 1000;
@@ -52,7 +52,7 @@
running = true;
const start = performance.now();
try {
const result = await source.query<Record<string, unknown>>(finalSql);
const result = await source.query<Record<string, unknown>>(finalSql, [], "SQL tab");
execTime = (performance.now() - start).toFixed(1);
rows = result.slice(0, MAX_ROWS);
columns = rows.length > 0 ? Object.keys(rows[0]) : [];
@@ -81,9 +81,6 @@
runPreset(schemaPreset.sql);
});
function formatBytes(n: number): string {
return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`;
}
</script>
<div class="mx-auto max-w-[900px]">
@@ -133,9 +130,12 @@
{#if execTime !== null}
<span class="text-sm text-ink-muted">{rows.length} kết quả · {execTime}ms</span>
{/if}
{#if db}
<!-- What this session has actually pulled over the network. -->
<span class="text-sm text-ink-subtle">Đã tải: {formatBytes(db.bytesRead)}</span>
{#if db?.lastCost}
<!-- What this query cost, then what the session has cost so far. -->
<span class="text-sm text-ink-subtle">
Truy vấn này: {db.lastCost.requests} yêu cầu · {formatBytes(db.lastCost.bytes)}
· Phiên: {db.requests} yêu cầu · {formatBytes(db.bytesRead)}
</span>
{/if}
</div>
</form>
+11 -6
View File
@@ -38,15 +38,20 @@ export async function searchByName(db: RemoteDatabase, query: string): Promise<S
WHERE w.word >= ? AND w.word < ?${filters.length ? " AND " + filters.join(" AND ") : ""}
LIMIT ${MAX_RESULTS}`;
return db.query<Student>(sql, [seek, upperBound(seek), ...others.map((w) => `% ${escapeLike(w)}%`)]);
return db.query<Student>(
sql,
[seek, upperBound(seek), ...others.map((w) => `% ${escapeLike(w)}%`)],
`name ${JSON.stringify(words.join(" "))} seeking ${JSON.stringify(seek)}`,
);
}
/** Exact lookup by exam number: a primary-key seek, a few pages. */
export async function lookupExamId(db: RemoteDatabase, id: string): Promise<Student[]> {
return db.query<Student>("SELECT * FROM student WHERE so_bao_danh = ? LIMIT ?", [
normaliseExamId(id),
MAX_RESULTS,
]);
return db.query<Student>(
"SELECT * FROM student WHERE so_bao_danh = ? LIMIT ?",
[normaliseExamId(id), MAX_RESULTS],
`exam id ${JSON.stringify(normaliseExamId(id))}`,
);
}
/** Fold to ASCII and split into words, the same shape name_word was built in. */
@@ -66,7 +71,7 @@ async function rarest(db: RemoteDatabase, words: string[]): Promise<string> {
.join(" UNION ALL ");
const params = words.flatMap((w) => [w, w, upperBound(w)]);
const counts = await db.query<{ word: string; n: number }>(sql, params);
const counts = await db.query<{ word: string; n: number }>(sql, params, "word frequencies");
let best = words[0];
let bestN = Infinity;
for (const { word, n } of counts) {
+64 -7
View File
@@ -1,4 +1,4 @@
import { createDbWorker, type WorkerHttpvfs } from "sql.js-httpvfs";
import { createDbWorker, type SqliteStats, type WorkerHttpvfs } from "sql.js-httpvfs";
import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url";
import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url";
@@ -25,6 +25,9 @@ export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024;
/** What the SQL tab gets once the user has accepted the cost of a scan. */
export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024;
/** What one query cost over the network. */
export type QueryCost = { requests: number; bytes: number; ms: number };
/**
* One remotely-paged database, with the load state the UI needs.
*
@@ -35,12 +38,18 @@ export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024;
export class RemoteDatabase {
ready = $state(false);
error = $state<string | null>(null);
/** Bytes fetched so far, refreshed after every query. */
/** Bytes fetched by this database so far, prefetch included. */
bytesRead = $state(0);
/** HTTP range requests issued so far. */
requests = $state(0);
/** What the most recent query cost on its own. */
lastCost = $state<QueryCost | null>(null);
#worker: WorkerHttpvfs | null = null;
#opening: Promise<WorkerHttpvfs>;
#closed = false;
// Previous cumulative reading, so a query's own cost is a subtraction.
#seen = { requests: 0, bytes: 0 };
constructor(
readonly url: string,
@@ -50,6 +59,7 @@ export class RemoteDatabase {
}
async #open(): Promise<WorkerHttpvfs> {
const opened = performance.now();
try {
const worker = await createDbWorker(
[{ from: "inline", config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES } }],
@@ -60,6 +70,9 @@ export class RemoteDatabase {
if (this.#closed) throw new Error("closed");
this.#worker = worker;
this.ready = true;
// Opening is not free either: the header and schema pages are read before
// any query runs, and that shows up in every later session total.
await this.#account(worker, `open ${this.url}`, performance.now() - opened);
return worker;
} catch (err) {
if (!this.#closed) {
@@ -70,21 +83,55 @@ export class RemoteDatabase {
}
}
/** Run a query and return its rows as objects. */
async query<T>(sql: string, params: unknown[] = []): Promise<T[]> {
/**
* Run a query and return its rows as objects.
*
* `label` names the query in the console trace — the only way to see what a
* search actually costs, since the byte count depends on how much the read
* heads prefetched, not just on the pages the plan needed.
*/
async query<T>(sql: string, params: unknown[] = [], label?: string): Promise<T[]> {
const worker = this.#worker ?? (await this.#opening);
// Comlink erases the generic when it proxies the method across the worker
// boundary, so the row type is asserted here rather than inferred.
const run = worker.db.query as unknown as (sql: string, ...params: unknown[]) => Promise<T[]>;
const started = performance.now();
try {
return await run(sql, ...params);
} finally {
// Comlink proxies the property, so this is a round trip; worth it because
// the number is the only honest feedback about what a query cost.
this.bytesRead = await worker.worker.bytesRead;
await this.#account(worker, label ?? firstLine(sql), performance.now() - started);
}
}
/**
* Read the cumulative counters and report the delta.
*
* getStats() rather than the worker's `bytesRead`: that one is the budget
* counter and resets itself to zero when a query trips the ceiling, so it
* would under-report exactly when the number matters most.
*/
async #account(worker: WorkerHttpvfs, label: string, ms: number) {
const read = worker.worker.getStats as unknown as () => Promise<SqliteStats | null>;
const stats = await read().catch(() => null);
if (!stats) return;
const cost: QueryCost = {
requests: stats.totalRequests - this.#seen.requests,
bytes: stats.totalFetchedBytes - this.#seen.bytes,
ms,
};
this.#seen = { requests: stats.totalRequests, bytes: stats.totalFetchedBytes };
this.requests = stats.totalRequests;
this.bytesRead = stats.totalFetchedBytes;
this.lastCost = cost;
console.info(
`[httpvfs] ${label}${cost.requests} request(s), ${formatBytes(cost.bytes)}, ${ms.toFixed(0)} ms` +
` · session ${stats.totalRequests} request(s), ${formatBytes(stats.totalFetchedBytes)}` +
` of ${formatBytes(stats.totalBytes)}`,
);
}
/**
* Drop this database. createDbWorker owns the Worker and exposes no handle to
* it, so the thread outlives this call; a page creates at most one per
@@ -106,3 +153,13 @@ export function isBudgetError(err: unknown): boolean {
function message(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
/** Enough of a query to recognise it in the console. */
function firstLine(sql: string): string {
const line = sql.trim().split("\n")[0];
return line.length > 70 ? `${line.slice(0, 70)}` : line;
}
export function formatBytes(n: number): string {
return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`;
}