mirror of
https://github.com/tiennm99/miti99bot.git
synced 2026-09-14 18:20:48 +00:00
refactor(doantu): swap Workers AI bge-m3 for hosted phow2sim HTTP API
Doantu now mirrors semantle's pre-Workers-AI shape: a thin fetch wrapper around /random + /similarity on https://phow2sim.sg.miti99.com (overridable via PHOW2SIM_API_URL). Drops the local Viet22K wordlist + build script — the service owns vocabulary now. Promotes commands from protected to public so they show up in Telegram's native / menu.
This commit is contained in:
+1
-2
@@ -9,10 +9,9 @@
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "wrangler dev",
|
||||
"build": "npm run build:wordle-data && npm run build:semantle-words && npm run build:doantu-words",
|
||||
"build": "npm run build:wordle-data && npm run build:semantle-words",
|
||||
"build:wordle-data": "node scripts/build-wordle-data.js",
|
||||
"build:semantle-words": "node scripts/build-semantle-words.js",
|
||||
"build:doantu-words": "node scripts/build-doantu-words.js",
|
||||
"scrape:loldle-data": "node scripts/scrape-loldle-data.js",
|
||||
"deploy": "npm run build && wrangler deploy && npm run db:migrate && npm run register",
|
||||
"db:migrate": "node scripts/migrate.js",
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file build-doantu-words — fetches the middle-sized Vietnamese wordlist
|
||||
* (Viet22K) from duyet/vietnamese-wordlist and writes it to
|
||||
* src/modules/doantu/words-data.js as a static ES-module array.
|
||||
*
|
||||
* Source is an alphabetically-sorted Unicode dictionary, one word or phrase
|
||||
* per line. We normalize only (trim, lowercase, dedupe); NO length/char
|
||||
* filtering — ConceptNet's verify-and-fallback in api-client handles
|
||||
* unusable picks at round start.
|
||||
*
|
||||
* Source: https://github.com/duyet/vietnamese-wordlist
|
||||
* Credits: Ho Ngoc Duc — Vietnamese word list (GPL).
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-doantu-words.js
|
||||
*/
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
// Size options from the upstream repo: Viet11K / Viet22K / Viet39K / Viet74K.
|
||||
// Viet22K is a good balance — enough variety, not overwhelmed by archaic terms.
|
||||
const SOURCE_URL = "https://raw.githubusercontent.com/duyet/vietnamese-wordlist/master/Viet22K.txt";
|
||||
|
||||
const root = resolve(import.meta.dirname, "..");
|
||||
const dst = resolve(root, "src/modules/doantu/words-data.js");
|
||||
|
||||
const res = await fetch(SOURCE_URL);
|
||||
if (!res.ok) throw new Error(`fetch failed: ${res.status} ${res.statusText}`);
|
||||
const text = await res.text();
|
||||
|
||||
// Normalize only: trim, lowercase, drop blanks, dedupe. Preserve source order.
|
||||
// Multi-word entries keep their spaces — the api-client converts to underscore
|
||||
// only at URL-build time, so the board still displays them naturally.
|
||||
const words = Array.from(
|
||||
new Set(
|
||||
text
|
||||
.split(/\r?\n/)
|
||||
.map((w) => w.trim().toLowerCase())
|
||||
.filter((w) => w.length > 0),
|
||||
),
|
||||
);
|
||||
|
||||
if (words.length === 0) throw new Error("no words parsed from source");
|
||||
|
||||
// JSON.stringify each word — safer than manual quoting for Vietnamese
|
||||
// diacritics and the occasional character that needs escaping.
|
||||
const body = words.map((w) => ` ${JSON.stringify(w)},`).join("\n");
|
||||
const out = [
|
||||
"// Auto-generated from https://github.com/duyet/vietnamese-wordlist (Viet22K.txt)",
|
||||
"// Credits: Ho Ngoc Duc — Vietnamese word list (GPL).",
|
||||
"// Normalized (lowercased + deduped) but otherwise unfiltered.",
|
||||
"// Regenerate with: node scripts/build-doantu-words.js",
|
||||
"export default [",
|
||||
body,
|
||||
"];",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
writeFileSync(dst, out);
|
||||
console.log(`wrote ${dst} (${words.length} words)`);
|
||||
@@ -1,48 +1,43 @@
|
||||
# Doantu Module
|
||||
|
||||
Vietnamese "đoán từ" (guess-the-word) — same core mechanic as `semantle`,
|
||||
but targets come from a Vietnamese wordlist and similarity is computed
|
||||
with a multilingual embedding model. Unlimited guesses per round; solve
|
||||
on exact match (case-insensitive, diacritic-sensitive).
|
||||
but targets + similarity come from a Vietnamese-tuned embedding service.
|
||||
Unlimited guesses per round; solve on exact match (case-insensitive,
|
||||
diacritic-sensitive).
|
||||
|
||||
**Visibility: `protected`** — commands appear in `/help` but are hidden
|
||||
from Telegram's native `/` autocomplete menu while the module is still
|
||||
experimental.
|
||||
**Visibility: `public`** — commands appear in both `/help` and Telegram's
|
||||
native `/` autocomplete menu.
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Visibility | Description |
|
||||
|---------|-----------|-------------|
|
||||
| `/doantu` | protected | Show current board or submit a word guess |
|
||||
| `/doantu_giveup` | protected | Reveal the answer and end the round (next `/doantu` starts a fresh one) |
|
||||
| `/doantu_stats` | protected | Show per-subject stats |
|
||||
| `/doantu` | public | Show current board or submit a word guess |
|
||||
| `/doantu_giveup` | public | Reveal the answer and end the round (next `/doantu` starts a fresh one) |
|
||||
| `/doantu_stats` | public | Show per-subject stats |
|
||||
|
||||
Submit with `/doantu <word>` (e.g. `/doantu con chó`). Multi-syllable words
|
||||
with single spaces between them are accepted. `cá` and `ca` are different
|
||||
targets.
|
||||
targets. Out-of-vocabulary words don't count toward the guess tally.
|
||||
Repeating a prior guess replies with a `🔁 already guessed` notice and is
|
||||
ignored (no cost, no stat inflation).
|
||||
|
||||
## Data source
|
||||
|
||||
**Target + vocabulary:** [duyet/vietnamese-wordlist](https://github.com/duyet/vietnamese-wordlist)'s
|
||||
Viet22K list (~22k entries), lowercased and deduped. The same list is
|
||||
both the target pool and the vocabulary — OOV detection is `Set.has()`
|
||||
with no upstream call. License: GPL-2.0 (Ho Ngoc Duc).
|
||||
Regenerate with `node scripts/build-doantu-words.js`.
|
||||
Target words + similarity scores come from our self-hosted **phow2sim**
|
||||
instance (default: `https://phow2sim.sg.miti99.com`). Wraps two endpoints:
|
||||
|
||||
**Similarity:** `@cf/baai/bge-m3` multilingual text embeddings via the
|
||||
`env.AI` binding. Chosen over the English-only `bge-small-en-v1.5`
|
||||
because that model's tokenizer shreds Vietnamese diacritics into noisy
|
||||
byte-level subwords. Each in-vocab guess runs one inference call
|
||||
batching target + guess (1024-dim vectors); the module scores them with
|
||||
local cosine similarity.
|
||||
- `GET /random` — pick a secret Vietnamese word at round start.
|
||||
- `GET /similarity?a=…&b=…` — cosine similarity + canonical forms +
|
||||
`in_vocab_a` / `in_vocab_b` flags.
|
||||
|
||||
Override the base URL for local dev via `PHOW2SIM_API_URL`.
|
||||
|
||||
## Architecture
|
||||
|
||||
- `api-client.js` — Workers AI wrapper: `randomWord()` picks from the
|
||||
local pool, `similarity(a, b)` calls `env.AI.run()` and returns
|
||||
`{ in_vocab_b, similarity }`. `UpstreamError` on inference failure.
|
||||
- `words-data.js` — auto-generated Viet22K dictionary.
|
||||
- `wordlist.js` — one-function module exposing `randomLine()`.
|
||||
- `api-client.js` — thin `fetch` wrapper around `/random` and
|
||||
`/similarity`. 5 s timeout; `UpstreamError` carries HTTP status + body
|
||||
snippet on failure.
|
||||
- `state.js` — KV persistence for game + stats. Same shape as semantle.
|
||||
- `lookup.js` — guess normalization + shape validation. Accepts Unicode
|
||||
letters + combining marks + single internal spaces.
|
||||
@@ -50,6 +45,9 @@ local cosine similarity.
|
||||
to semantle/format.js — score display is language-agnostic).
|
||||
- `render.js` — Telegram HTML `<pre>` monospace board with a 🇻🇳 header.
|
||||
- `handlers.js` — subject resolution + the three command entry points.
|
||||
Fast-path dedup (exact text OR prior canonical) skips wasted API calls
|
||||
on repeat guesses; post-API dedup catches different inputs that
|
||||
canonicalize to the same token.
|
||||
|
||||
Near-clone of the semantle sibling — kept separate per the repo's
|
||||
one-module-per-game convention rather than factoring out a shared base.
|
||||
@@ -65,13 +63,15 @@ KV namespace prefix: `doantu:`
|
||||
| `game:<subject>` | `{ target, startedAt, solved, guesses[] }` — active round (TTL 7 days). |
|
||||
| `stats:<subject>` | `{ played, solved, totalGuesses, bestGuessCount, lastResultAt }` |
|
||||
|
||||
Each `guesses[]` entry is `{ word, canonical, similarity }`.
|
||||
|
||||
## Config
|
||||
|
||||
No env vars. Model defaults to `@cf/baai/bge-m3`; override with
|
||||
`createClient(env.AI, { model: "..." })` in a test or alternative deploy.
|
||||
| Env var | Default | Purpose |
|
||||
|---------|---------|---------|
|
||||
| `PHOW2SIM_API_URL` | `https://phow2sim.sg.miti99.com` | Base URL for the phow2sim service. |
|
||||
|
||||
## Credits
|
||||
|
||||
- Embeddings: [`@cf/baai/bge-m3`](https://developers.cloudflare.com/workers-ai/models/bge-m3/) on Cloudflare Workers AI (multilingual).
|
||||
- Wordlist: [duyet/vietnamese-wordlist](https://github.com/duyet/vietnamese-wordlist) by Ho Ngoc Duc (GPL-2.0).
|
||||
- Similarity backend: self-hosted `phow2sim` (Vietnamese word2vec/PhoBERT-style).
|
||||
- Game concept: [Semantle](https://semantle.com/) by David Turner.
|
||||
|
||||
@@ -1,29 +1,16 @@
|
||||
/**
|
||||
* @file Cloudflare Workers AI client for the doantu module (Vietnamese semantle).
|
||||
* @file phow2sim HTTP API client for the doantu module (Vietnamese semantle).
|
||||
*
|
||||
* Mirrors semantle/api-client.js but uses `@cf/baai/bge-m3` — BAAI's
|
||||
* multilingual embedding model — because the English-only BGE variants
|
||||
* can't produce meaningful Vietnamese vectors (their tokenizer is
|
||||
* English-centric and Vietnamese diacritics get shredded into noisy
|
||||
* byte-level subwords).
|
||||
* Wraps two endpoints:
|
||||
* GET /random → pick a secret Vietnamese word at round start
|
||||
* GET /similarity → cosine similarity between target and guess per turn
|
||||
*
|
||||
* Vocabulary: the curated `words-data.js` list (duyet/vietnamese-wordlist
|
||||
* Viet22K) doubles as the in/out-of-vocabulary set. Lookups are O(1) via
|
||||
* Set.has(), so OOV detection needs no extra round-trip.
|
||||
*
|
||||
* The returned `similarity(a, b)` shape matches the semantle sibling so
|
||||
* handlers/render/state can be reused unchanged.
|
||||
* Stateless. No caching layer — phow2sim is cheap enough, and caching per-pair
|
||||
* scores in KV would pollute the namespace without measurable gain.
|
||||
*/
|
||||
|
||||
import { randomLine } from "./wordlist.js";
|
||||
import WORDS from "./words-data.js";
|
||||
|
||||
// BGE-M3: multilingual (194 languages incl. Vietnamese), 1024 dimensions,
|
||||
// ~1,075 Neurons per M input tokens — cheaper than bge-small-en-v1.5.
|
||||
const DEFAULT_MODEL = "@cf/baai/bge-m3";
|
||||
|
||||
// O(1) membership lookup for OOV detection. Built once per isolate.
|
||||
const VOCAB = new Set(WORDS);
|
||||
const DEFAULT_TIMEOUT_MS = 5000;
|
||||
const USER_AGENT = "miti99bot/doantu";
|
||||
|
||||
export class UpstreamError extends Error {
|
||||
/** @param {string} message @param {{status?: number, body?: string, cause?: unknown}} [meta] */
|
||||
@@ -36,70 +23,71 @@ export class UpstreamError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
function cosineSimilarity(a, b) {
|
||||
if (!a || !b || a.length !== b.length) return null;
|
||||
let dot = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
function buildUrl(base, path, params) {
|
||||
const normalized = String(base).replace(/\/+$/, "");
|
||||
const url = new URL(`${normalized}${path}`);
|
||||
for (const [k, v] of Object.entries(params)) {
|
||||
if (v === undefined || v === null) continue;
|
||||
url.searchParams.set(k, String(v));
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
async function fetchJson(url, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
headers: { "User-Agent": USER_AGENT, Accept: "application/json" },
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
throw new UpstreamError("phow2sim fetch failed", { cause: err });
|
||||
}
|
||||
clearTimeout(timer);
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
throw new UpstreamError(`phow2sim HTTP ${res.status}`, {
|
||||
status: res.status,
|
||||
body: text.slice(0, 500),
|
||||
});
|
||||
}
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (err) {
|
||||
throw new UpstreamError("phow2sim non-JSON response", { cause: err });
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? null : dot / denom;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{ run: (model: string, input: { text: string[] }) => Promise<{ data: number[][] }> }} ai
|
||||
* — Workers AI binding (`env.AI`). Tests pass a fake with the same `.run()` shape.
|
||||
* @param {{ model?: string }} [opts]
|
||||
* @param {string} apiBase — e.g. "https://phow2sim.sg.miti99.com"
|
||||
* @param {{ timeoutMs?: number }} [opts]
|
||||
*/
|
||||
export function createClient(ai, { model = DEFAULT_MODEL } = {}) {
|
||||
if (!ai || typeof ai.run !== "function") {
|
||||
throw new TypeError("createClient: ai binding with .run(model, input) is required");
|
||||
}
|
||||
|
||||
async function embedPair(a, b) {
|
||||
let resp;
|
||||
try {
|
||||
resp = await ai.run(model, { text: [a, b] });
|
||||
} catch (err) {
|
||||
throw new UpstreamError("workers-ai embedding failed", { cause: err });
|
||||
}
|
||||
const data = resp?.data;
|
||||
if (!Array.isArray(data) || data.length < 2) {
|
||||
throw new UpstreamError("workers-ai returned malformed embedding payload");
|
||||
}
|
||||
return [data[0], data[1]];
|
||||
}
|
||||
|
||||
export function createClient(apiBase, { timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
||||
return {
|
||||
/**
|
||||
* Pick a target word from the local Vietnamese pool. The pool IS the
|
||||
* vocabulary, so every pick is trivially verified.
|
||||
* @returns {Promise<{ word: string, verified: boolean }>}
|
||||
* Pick a random vocab word matching filters.
|
||||
* @param {Record<string, string|number|boolean>} [filters]
|
||||
* @returns {Promise<{ word: string, rank?: number }>}
|
||||
*/
|
||||
async randomWord() {
|
||||
return { word: randomLine(), verified: true };
|
||||
randomWord(filters = {}) {
|
||||
return fetchJson(buildUrl(apiBase, "/random", filters), timeoutMs);
|
||||
},
|
||||
|
||||
/**
|
||||
* Cosine similarity between `a` (target) and `b` (guess). Uses the local
|
||||
* Vietnamese wordlist as vocabulary — unknown words return
|
||||
* `in_vocab_b: false` with `similarity: null` and skip inference.
|
||||
*
|
||||
* Cosine similarity between two words.
|
||||
* @param {string} a
|
||||
* @param {string} b
|
||||
* @returns {Promise<{
|
||||
* a: string, b: string,
|
||||
* canonical_a: string|null, canonical_b: string|null,
|
||||
* in_vocab_a: boolean, in_vocab_b: boolean,
|
||||
* similarity: number|null
|
||||
* }>}
|
||||
*/
|
||||
async similarity(a, b) {
|
||||
const base = { a, b, canonical_a: a, canonical_b: b, in_vocab_a: true };
|
||||
if (!VOCAB.has(b)) {
|
||||
return { ...base, in_vocab_b: false, similarity: null };
|
||||
}
|
||||
const [vecA, vecB] = await embedPair(a, b);
|
||||
const sim = cosineSimilarity(vecA, vecB);
|
||||
return { ...base, in_vocab_b: true, similarity: sim };
|
||||
similarity(a, b) {
|
||||
return fetchJson(buildUrl(apiBase, "/similarity", { a, b }), timeoutMs);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
* private chat → user id (per-user game)
|
||||
* group/supergroup chat → chat id (shared game)
|
||||
*
|
||||
* Commands (all protected — listed in /help but not pushed to the Telegram
|
||||
* native / menu):
|
||||
* Commands:
|
||||
* /doantu → show the board (or lazy-start a round)
|
||||
* /doantu <word> → submit a guess
|
||||
* /doantu_giveup → reveal target (next /doantu auto-starts fresh)
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
/**
|
||||
* @file Doantu module — Vietnamese semantle.
|
||||
*
|
||||
* Targets from a curated local wordlist (duyet/vietnamese-wordlist Viet22K —
|
||||
* same list doubles as the vocabulary for OOV detection). Similarity scores
|
||||
* come from cosine distance between `@cf/baai/bge-m3` multilingual embeddings
|
||||
* produced by the `env.AI` binding. All commands are `protected` — listed
|
||||
* in /help but hidden from Telegram's native / autocomplete menu.
|
||||
* Target words + cosine similarity come from our own hosted phow2sim instance
|
||||
* (default: https://phow2sim.sg.miti99.com). Override via env var
|
||||
* `PHOW2SIM_API_URL` for local dev or self-hosting.
|
||||
*/
|
||||
|
||||
import { createClient } from "./api-client.js";
|
||||
import { handleDoantu, handleGiveup, handleStats } from "./handlers.js";
|
||||
|
||||
const DEFAULT_API_URL = "https://phow2sim.sg.miti99.com";
|
||||
|
||||
/** @type {import("../../db/kv-store-interface.js").KVStore | null} */
|
||||
let db = null;
|
||||
/** @type {ReturnType<typeof createClient> | null} */
|
||||
@@ -21,24 +21,25 @@ const doantuModule = {
|
||||
name: "doantu",
|
||||
init: async ({ db: store, env }) => {
|
||||
db = store;
|
||||
client = createClient(env.AI);
|
||||
const base = env?.PHOW2SIM_API_URL || DEFAULT_API_URL;
|
||||
client = createClient(base);
|
||||
},
|
||||
commands: [
|
||||
{
|
||||
name: "doantu",
|
||||
visibility: "protected",
|
||||
visibility: "public",
|
||||
description: "Đoán từ — Vietnamese semantic word guessing (unlimited tries)",
|
||||
handler: (ctx) => handleDoantu(ctx, { db, client }),
|
||||
},
|
||||
{
|
||||
name: "doantu_giveup",
|
||||
visibility: "protected",
|
||||
visibility: "public",
|
||||
description: "Reveal the current doantu answer (auto-starts a fresh round)",
|
||||
handler: (ctx) => handleGiveup(ctx, { db, client }),
|
||||
},
|
||||
{
|
||||
name: "doantu_stats",
|
||||
visibility: "protected",
|
||||
visibility: "public",
|
||||
description: "Show your doantu stats",
|
||||
handler: (ctx) => handleStats(ctx, { db, client }),
|
||||
},
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
/**
|
||||
* @file Target-word pool for the doantu module (Vietnamese).
|
||||
*
|
||||
* Data lives in the sibling `words-data.js` (auto-generated by
|
||||
* `scripts/build-doantu-words.js` from duyet/vietnamese-wordlist's
|
||||
* Viet22K list, lowercased + deduped).
|
||||
*/
|
||||
|
||||
import LINES from "./words-data.js";
|
||||
|
||||
/** @returns {string} */
|
||||
export function randomLine() {
|
||||
return LINES[Math.floor(Math.random() * LINES.length)];
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,26 +1,11 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { UpstreamError, createClient } from "../../../src/modules/doantu/api-client.js";
|
||||
|
||||
/**
|
||||
* Build a deterministic 1024-dim vector from a seed so cosine scores are
|
||||
* reproducible in tests without hardcoding floats. bge-m3 produces 1024-dim
|
||||
* vectors; tests use the same width for realism.
|
||||
*/
|
||||
function fakeVector(seed, dim = 1024) {
|
||||
const out = new Array(dim);
|
||||
for (let i = 0; i < dim; i++) out[i] = Math.sin(seed * (i + 1));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal Workers AI binding fake. `impl(model, input)` returns the payload
|
||||
* `env.AI.run()` would normally resolve to.
|
||||
*/
|
||||
function fakeAi(impl) {
|
||||
return { run: vi.fn(impl) };
|
||||
}
|
||||
|
||||
describe("doantu/api-client", () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("UpstreamError", () => {
|
||||
it("stores status and body metadata", () => {
|
||||
const err = new UpstreamError("test", { status: 404, body: "not found" });
|
||||
@@ -38,107 +23,156 @@ describe("doantu/api-client", () => {
|
||||
});
|
||||
|
||||
describe("createClient", () => {
|
||||
it("throws without a valid AI binding", () => {
|
||||
expect(() => createClient(null)).toThrow(TypeError);
|
||||
expect(() => createClient({})).toThrow(TypeError);
|
||||
expect(() => createClient({ run: "not a function" })).toThrow(TypeError);
|
||||
it("randomWord builds correct URL with filters", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((url) => {
|
||||
expect(url).toContain("/random");
|
||||
expect(url).toContain("min_len=2");
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve('{"word":"chó"}'),
|
||||
});
|
||||
});
|
||||
const res = await client.randomWord({ min_len: 2 });
|
||||
expect(res.word).toBe("chó");
|
||||
});
|
||||
|
||||
it("similarity batches target + guess in a single run() call with bge-m3", async () => {
|
||||
const ai = fakeAi(async (_model, { text }) => ({
|
||||
shape: [text.length, 1024],
|
||||
data: text.map((_, i) => fakeVector(i + 1)),
|
||||
}));
|
||||
const client = createClient(ai);
|
||||
await client.similarity("chó", "mèo");
|
||||
expect(ai.run).toHaveBeenCalledTimes(1);
|
||||
const [model, input] = ai.run.mock.calls[0];
|
||||
expect(model).toBe("@cf/baai/bge-m3");
|
||||
expect(input).toEqual({ text: ["chó", "mèo"] });
|
||||
});
|
||||
|
||||
it("similarity returns cosine score for an in-vocab Vietnamese guess", async () => {
|
||||
const ai = fakeAi(async (_model, { text }) => ({
|
||||
data: text.map((_, i) => fakeVector(i + 1)),
|
||||
}));
|
||||
const client = createClient(ai);
|
||||
it("similarity builds URL with both words", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((url) => {
|
||||
expect(url).toContain("/similarity");
|
||||
expect(url).toMatch(/a=ch%C3%B3/);
|
||||
expect(url).toMatch(/b=m%C3%A8o/);
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
'{"a":"chó","b":"mèo","in_vocab_a":true,"in_vocab_b":true,"canonical_a":"chó","canonical_b":"mèo","similarity":0.42}',
|
||||
),
|
||||
});
|
||||
});
|
||||
const res = await client.similarity("chó", "mèo");
|
||||
expect(res.in_vocab_a).toBe(true);
|
||||
expect(res.in_vocab_b).toBe(true);
|
||||
expect(res.canonical_a).toBe("chó");
|
||||
expect(res.similarity).toBe(0.42);
|
||||
expect(res.canonical_b).toBe("mèo");
|
||||
expect(typeof res.similarity).toBe("number");
|
||||
expect(res.similarity).toBeGreaterThan(-1);
|
||||
expect(res.similarity).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('similarity accepts multi-syllable Vietnamese words in vocab ("a dua")', async () => {
|
||||
const ai = fakeAi(async () => ({ data: [fakeVector(1), fakeVector(2)] }));
|
||||
const client = createClient(ai);
|
||||
const res = await client.similarity("chó", "a dua");
|
||||
expect(res.in_vocab_b).toBe(true);
|
||||
expect(res.similarity).not.toBeNull();
|
||||
});
|
||||
|
||||
it("similarity returns 1 for identical vectors", async () => {
|
||||
const vec = fakeVector(7);
|
||||
const ai = fakeAi(async () => ({ data: [vec, vec] }));
|
||||
const client = createClient(ai);
|
||||
const res = await client.similarity("chó", "mèo");
|
||||
expect(res.similarity).toBeCloseTo(1, 10);
|
||||
});
|
||||
|
||||
it("similarity skips AI call for OOV guess and flags in_vocab_b:false", async () => {
|
||||
const ai = fakeAi(async () => ({ data: [fakeVector(1), fakeVector(2)] }));
|
||||
const client = createClient(ai);
|
||||
const res = await client.similarity("chó", "zzzkhôngcótrongtừđiển");
|
||||
expect(res.in_vocab_b).toBe(false);
|
||||
expect(res.similarity).toBe(null);
|
||||
expect(ai.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("similarity wraps AI.run rejection as UpstreamError", async () => {
|
||||
const ai = fakeAi(async () => {
|
||||
throw new Error("boom");
|
||||
it("URL-encodes multi-syllable Vietnamese guesses", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((url) => {
|
||||
expect(url).toMatch(/b=con\+ch%C3%B3|b=con%20ch%C3%B3/);
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () =>
|
||||
Promise.resolve(
|
||||
'{"a":"mèo","b":"con chó","in_vocab_a":true,"in_vocab_b":true,"similarity":0.3}',
|
||||
),
|
||||
});
|
||||
});
|
||||
const client = createClient(ai);
|
||||
await expect(client.similarity("chó", "mèo")).rejects.toMatchObject({
|
||||
await client.similarity("mèo", "con chó");
|
||||
expect(global.fetch).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws UpstreamError on non-2xx response", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: () => Promise.resolve("Internal Server Error"),
|
||||
}),
|
||||
);
|
||||
await expect(client.randomWord()).rejects.toMatchObject({
|
||||
name: "UpstreamError",
|
||||
status: 500,
|
||||
body: "Internal Server Error",
|
||||
});
|
||||
});
|
||||
|
||||
it("throws UpstreamError when response is not valid JSON", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve("not json at all"),
|
||||
}),
|
||||
);
|
||||
await expect(client.randomWord()).rejects.toMatchObject({
|
||||
name: "UpstreamError",
|
||||
});
|
||||
});
|
||||
|
||||
it("similarity throws UpstreamError on malformed payload", async () => {
|
||||
const ai = fakeAi(async () => ({ data: [fakeVector(1)] }));
|
||||
const client = createClient(ai);
|
||||
await expect(client.similarity("chó", "mèo")).rejects.toMatchObject({
|
||||
name: "UpstreamError",
|
||||
it("throws UpstreamError on fetch failure", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn(() => Promise.reject(new Error("network error")));
|
||||
await expect(client.randomWord()).rejects.toThrow("phow2sim fetch failed");
|
||||
});
|
||||
|
||||
it("truncates response body to 500 chars on non-OK", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 50 });
|
||||
const longBody = "x".repeat(600);
|
||||
global.fetch = vi.fn(() =>
|
||||
Promise.resolve({
|
||||
ok: false,
|
||||
status: 400,
|
||||
text: () => Promise.resolve(longBody),
|
||||
}),
|
||||
);
|
||||
try {
|
||||
await client.randomWord();
|
||||
} catch (err) {
|
||||
expect(err.body.length).toBe(500);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes User-Agent header identifying doantu", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((_, opts) => {
|
||||
expect(opts.headers["User-Agent"]).toContain("miti99bot");
|
||||
expect(opts.headers["User-Agent"]).toContain("doantu");
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve('{"word":"chó"}'),
|
||||
});
|
||||
});
|
||||
await client.randomWord();
|
||||
});
|
||||
|
||||
it("similarity returns null score when a vector norm is zero", async () => {
|
||||
const zero = new Array(1024).fill(0);
|
||||
const ai = fakeAi(async () => ({ data: [zero, fakeVector(1)] }));
|
||||
const client = createClient(ai);
|
||||
const res = await client.similarity("chó", "mèo");
|
||||
expect(res.in_vocab_b).toBe(true);
|
||||
expect(res.similarity).toBe(null);
|
||||
it("includes Accept header", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((_, opts) => {
|
||||
expect(opts.headers.Accept).toBe("application/json");
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve('{"word":"chó"}'),
|
||||
});
|
||||
});
|
||||
await client.randomWord();
|
||||
});
|
||||
|
||||
it("randomWord returns a verified pick from the local pool", async () => {
|
||||
const ai = fakeAi(async () => ({ data: [] }));
|
||||
const client = createClient(ai);
|
||||
const res = await client.randomWord();
|
||||
expect(typeof res.word).toBe("string");
|
||||
expect(res.word.length).toBeGreaterThan(0);
|
||||
expect(res.verified).toBe(true);
|
||||
expect(ai.run).not.toHaveBeenCalled();
|
||||
it("handles trailing slashes in API base URL", async () => {
|
||||
const client = createClient("https://api.test///", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((url) => {
|
||||
expect(url.startsWith("https://api.test/")).toBe(true);
|
||||
expect(url.startsWith("https://api.test////")).toBe(false);
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve('{"word":"chó"}'),
|
||||
});
|
||||
});
|
||||
await client.randomWord();
|
||||
});
|
||||
|
||||
it("supports model override via options", async () => {
|
||||
const ai = fakeAi(async () => ({ data: [fakeVector(1), fakeVector(2)] }));
|
||||
const client = createClient(ai, { model: "@cf/baai/bge-large-en-v1.5" });
|
||||
await client.similarity("chó", "mèo");
|
||||
expect(ai.run.mock.calls[0][0]).toBe("@cf/baai/bge-large-en-v1.5");
|
||||
it("filters out undefined/null params", async () => {
|
||||
const client = createClient("https://api.test", { timeoutMs: 100 });
|
||||
global.fetch = vi.fn((url) => {
|
||||
expect(url).not.toContain("min_len=");
|
||||
expect(url).not.toContain("max_len=");
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
text: () => Promise.resolve('{"word":"chó"}'),
|
||||
});
|
||||
});
|
||||
await client.randomWord({ min_len: undefined, max_len: null });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user