From 8dd17acd4fa2088fa2e423063550a089e791cfd0 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Thu, 23 Apr 2026 10:05:50 +0700 Subject: [PATCH] feat: initial phow2sim service Tiny FastAPI service over PhoW2V Vietnamese word vectors. Mirrors word2sim's endpoint shapes (/similarity /neighbors /vocab /random) so clients can swap URLs without code changes. - Auto-downloads VinAI's PhoW2V on first boot, caches binary .bin for ~5x faster restarts - Viet-aware canonicalizer: exact -> lowercase -> space-to-underscore - Supports both word (compound) and syllable variants via env - Unicode-aware random-word filter accepts diacritics, rejects digits/punct --- .dockerignore | 9 +++ .gitignore | 9 +++ Dockerfile | 26 +++++++ README.md | 138 +++++++++++++++++++++++++++++++++ app/main.py | 141 +++++++++++++++++++++++++++++++++ app/vectors.py | 154 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 20 +++++ requirements.txt | 4 + scripts/download-phow2v.sh | 52 +++++++++++++ 9 files changed, 553 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/main.py create mode 100644 app/vectors.py create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100755 scripts/download-phow2v.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8f07d48 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.gitignore +__pycache__ +*.pyc +.venv +venv +models +*.zip +README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf5bca3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__/ +*.pyc +.venv/ +venv/ +.env +.DS_Store +models/ +*.zip +*.txt.bin diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2084e1b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl unzip ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +# Defaults point at PhoW2V word-300d from VinAI's public mirror. +# Override MODEL_URL/MODEL_PATH to switch variants (syllables, 100d). +ENV MODEL_URL=https://public.vinai.io/word2vec_vi_words_300dims.zip \ + MODEL_PATH=/data/phow2v/word2vec_vi_words_300dims.txt \ + MODEL_VARIANT=word \ + PORT=8000 + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=3 \ + CMD curl -fsS http://localhost:8000/health || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..7ff348f --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# phow2sim + +Tiny HTTP service that returns Vietnamese word2vec similarity and nearest +neighbors — Vietnamese sibling of [`word2sim`](../word2sim). Same endpoint +shapes; swap URLs and it's a drop-in replacement. + +Backed by [**PhoW2V**](https://github.com/VinAIResearch/PhoW2V) (VinAI), the +largest pretrained Vietnamese word vectors available. Chosen over PhoBERT +for this purpose because word2vec's similarity distribution is wide +enough to drive a Semantle-style warmth meter, whereas raw transformer +embeddings saturate at the top. + +## Stack + +- FastAPI + uvicorn +- gensim (loads PhoW2V `.txt` files; caches a binary `.bin` alongside for 5× faster restarts) + +## Variants + +PhoW2V ships in four flavors. Pick one per deployment. + +| Variant | Dims | Size | Best for | +|---|---|---|---| +| `word-100` | 100 | ~400MB | low-RAM hosts, compound-aware | +| `word-300` | 300 | ~1.2GB | **default** — best quality, compound-aware | +| `syllable-100`| 100 | ~50MB | single-syllable guesses, tiny footprint | +| `syllable-300`| 300 | ~150MB | single-syllable guesses, richer vectors | + +The "word" variants expect underscore-joined compounds (`sinh_viên`); +the "syllable" variants have no multi-token keys. The canonicalizer +tries both forms, but the client should pre-segment for the word variant +if it wants reliable coverage of compounds. + +## Endpoints + +| Method | Path | Purpose | +|---|---|---| +| GET | `/health` | liveness probe | +| GET | `/similarity?a=X&b=Y` | cosine similarity between two keys | +| GET | `/neighbors?word=X&topn=10` | nearest-neighbor keys with scores | +| GET | `/vocab?word=X` | check in-vocab; return canonical form | +| GET | `/random` | random vocab key, filtered for game-friendliness | + +Response shape is identical to word2sim. + +### Examples + +```bash +curl 'http://localhost:8001/similarity?a=con_chó&b=con_mèo' +# {"a":"con_chó","b":"con_mèo","canonical_a":"con_chó","canonical_b":"con_mèo", +# "in_vocab_a":true,"in_vocab_b":true,"similarity":0.78} + +curl 'http://localhost:8001/neighbors?word=đại_học&topn=5' + +curl 'http://localhost:8001/vocab?word=con%20ch%C3%B3' # "con chó" → tries "con_chó" +# {"word":"con chó","canonical":"con_chó","in_vocab":true} + +curl 'http://localhost:8001/random?min_rank=500&max_rank=20000&min_len=3&max_len=12' +``` + +Out-of-vocab returns `in_vocab:false` and `similarity:null`. Lookup +tries exact → lowercase → space-to-underscore variants. + +## Quick start + +```bash +docker compose up --build +# First boot downloads ~1.2GB (word-300d) into the `phow2v-cache` volume. +# Model parse ~60s. A binary cache is written on first success so later +# restarts take ~10s. +``` + +Health check start period is 10 min to cover the download + parse. + +## Switching variant + +Edit `docker-compose.yml` or pass env: + +```bash +MODEL_URL=https://public.vinai.io/word2vec_vi_syllables_100dims.zip \ +MODEL_PATH=/data/phow2v/word2vec_vi_syllables_100dims.txt \ +MODEL_VARIANT=syllable \ +docker compose up --build +``` + +Delete the `phow2v-cache` volume when switching, otherwise the stale +`.bin` from the previous variant will load instead. + +## Manual model population + +Skip the auto-download if you want to prepare the volume ahead of time: + +```bash +./scripts/download-phow2v.sh word 300 # word-300d into ./models +# then mount ./models as /data/phow2v in docker-compose.yml +``` + +## Config (env vars) + +| Var | Default | Meaning | +|---|---|---| +| `MODEL_URL` | `https://public.vinai.io/word2vec_vi_words_300dims.zip` | fetched on first boot if `MODEL_PATH` absent | +| `MODEL_PATH` | `/data/phow2v/word2vec_vi_words_300dims.txt` | where the text-format vectors live | +| `MODEL_VARIANT` | `word` | declarative hint for the caller; `word` or `syllable` | + +## Using from doantu (miti99bot) + +The Cloudflare Worker module's `api-client.js` already produces the same +response shape. Replace `embedPair` + local cosine with a single `fetch`: + +```js +const url = `${env.PHOW2SIM_URL}/similarity?a=${encodeURIComponent(a)}&b=${encodeURIComponent(b)}`; +const resp = await fetch(url, { headers: { Authorization: `Bearer ${env.PHOW2SIM_TOKEN}` } }); +return await resp.json(); // { in_vocab_a, in_vocab_b, similarity, ... } +``` + +Auth is **not** built-in here — add a reverse proxy (Caddy, Cloudflare +Tunnel, or nginx) in front that checks a bearer token before passing +through. The service itself trusts its caller. + +## Project layout + +``` +phow2sim/ +├── app/ +│ ├── main.py # FastAPI routes +│ └── vectors.py # PhoW2V loader + canonicalize + similarity/neighbors/random +├── scripts/ +│ └── download-phow2v.sh +├── Dockerfile +├── docker-compose.yml +└── requirements.txt +``` + +## Credits + +- Vectors: [PhoW2V](https://github.com/VinAIResearch/PhoW2V) by VinAI Research (research license — see their repo). +- API shape: sibling of [`word2sim`](../word2sim). diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..8163555 --- /dev/null +++ b/app/main.py @@ -0,0 +1,141 @@ +"""FastAPI entry point for phow2sim. + +Read-only endpoints over PhoW2V: pairwise similarity, nearest neighbors, +vocab lookup, random word. Stateless. Response shapes mirror word2sim so +callers can swap the two services via URL alone. +""" + +from __future__ import annotations + +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException, Query +from pydantic import BaseModel + +from app.vectors import canonicalize, load_model, neighbors, random_word, similarity + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + # Force download + model load before accepting traffic (PhoW2V word-300d ~60s cold). + load_model() + yield + + +app = FastAPI(title="phow2sim", version="0.1.0", lifespan=lifespan) + + +class SimilarityResponse(BaseModel): + a: str + b: str + canonical_a: str | None + canonical_b: str | None + in_vocab_a: bool + in_vocab_b: bool + similarity: float | None # null iff either side is out-of-vocab + + +class NeighborEntry(BaseModel): + word: str + similarity: float + + +class NeighborsResponse(BaseModel): + word: str + canonical: str | None + in_vocab: bool + neighbors: list[NeighborEntry] + + +class VocabResponse(BaseModel): + word: str + canonical: str | None + in_vocab: bool + + +class RandomWordResponse(BaseModel): + word: str + rank: int + + +@app.get("/health") +def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.get("/similarity", response_model=SimilarityResponse) +def get_similarity( + a: str = Query(min_length=1, max_length=64), + b: str = Query(min_length=1, max_length=64), +) -> SimilarityResponse: + kv = load_model() + canonical_a = canonicalize(kv, a) + canonical_b = canonicalize(kv, b) + score = ( + similarity(kv, canonical_a, canonical_b) + if canonical_a and canonical_b + else None + ) + return SimilarityResponse( + a=a, + b=b, + canonical_a=canonical_a, + canonical_b=canonical_b, + in_vocab_a=canonical_a is not None, + in_vocab_b=canonical_b is not None, + similarity=score, + ) + + +@app.get("/neighbors", response_model=NeighborsResponse) +def get_neighbors( + word: str = Query(min_length=1, max_length=64), + topn: int = Query(default=10, ge=1, le=1000), +) -> NeighborsResponse: + kv = load_model() + canonical = canonicalize(kv, word) + if canonical is None: + return NeighborsResponse( + word=word, canonical=None, in_vocab=False, neighbors=[] + ) + entries = [ + NeighborEntry(word=w, similarity=s) for w, s in neighbors(kv, canonical, topn) + ] + return NeighborsResponse( + word=word, canonical=canonical, in_vocab=True, neighbors=entries + ) + + +@app.get("/vocab", response_model=VocabResponse) +def get_vocab(word: str = Query(min_length=1, max_length=64)) -> VocabResponse: + kv = load_model() + canonical = canonicalize(kv, word) + return VocabResponse(word=word, canonical=canonical, in_vocab=canonical is not None) + + +@app.get("/random", response_model=RandomWordResponse) +def get_random( + min_rank: int = Query(default=100, ge=0), + max_rank: int = Query(default=50000, ge=1), + alpha_only: bool = Query(default=True), + min_len: int = Query(default=2, ge=1, le=64), + max_len: int = Query(default=20, ge=1, le=64), +) -> RandomWordResponse: + if min_rank >= max_rank: + raise HTTPException(status_code=400, detail="min_rank must be < max_rank") + if min_len > max_len: + raise HTTPException(status_code=400, detail="min_len must be <= max_len") + kv = load_model() + word = random_word( + kv, + min_rank=min_rank, + max_rank=max_rank, + alpha_only=alpha_only, + min_len=min_len, + max_len=max_len, + ) + if word is None: + raise HTTPException( + status_code=503, detail="no word matched filter; loosen the constraints" + ) + return RandomWordResponse(word=word, rank=kv.key_to_index[word]) diff --git a/app/vectors.py b/app/vectors.py new file mode 100644 index 0000000..75a4c9c --- /dev/null +++ b/app/vectors.py @@ -0,0 +1,154 @@ +"""PhoW2V model loader and similarity primitives. + +PhoW2V (VinAI) ships as word2vec-text format (.txt), available in four +variants: word/syllable × 100/300 dims. Text format is slow to parse on +first boot (~30-60s for the 300d word model), so we cache a binary .bin +alongside the .txt after the first successful load — subsequent starts +use the fast binary path. + +Tokenization matters. The "word" variant expects underscore-joined +compounds ("sinh_viên"); the "syllable" variant expects single syllables +("sinh", "viên"). Callers must normalize to match before querying. +""" + +from __future__ import annotations + +import os +import random as _random +import unicodedata +import urllib.request +import zipfile +from pathlib import Path +from typing import Optional + +from gensim.models import KeyedVectors + +_MODEL: Optional[KeyedVectors] = None + + +def _download_and_extract(url: str, target_txt: Path) -> None: + """Fetch a PhoW2V zip and extract its .txt into target_txt.""" + target_txt.parent.mkdir(parents=True, exist_ok=True) + zip_path = target_txt.with_suffix(".zip") + urllib.request.urlretrieve(url, zip_path) + with zipfile.ZipFile(zip_path) as zf: + txt_members = [m for m in zf.namelist() if m.endswith(".txt")] + if not txt_members: + raise RuntimeError(f"no .txt file inside {url}") + # Flatten into target_txt regardless of archive's internal layout. + with zf.open(txt_members[0]) as src, open(target_txt, "wb") as dst: + dst.write(src.read()) + zip_path.unlink(missing_ok=True) + + +def load_model() -> KeyedVectors: + """Return the singleton KeyedVectors, loading (and downloading) on first call.""" + global _MODEL + if _MODEL is not None: + return _MODEL + + txt_path = Path(os.environ["MODEL_PATH"]) + bin_cache = txt_path.with_suffix(".bin") + + # Prefer the cached binary form for ~5x faster cold start. + if bin_cache.exists(): + _MODEL = KeyedVectors.load_word2vec_format(str(bin_cache), binary=True) + return _MODEL + + if not txt_path.exists(): + url = os.environ.get("MODEL_URL") + if not url: + raise FileNotFoundError( + f"MODEL_PATH {txt_path} missing and no MODEL_URL set for auto-download" + ) + _download_and_extract(url, txt_path) + + _MODEL = KeyedVectors.load_word2vec_format(str(txt_path), binary=False) + # Persist the fast-load cache next to the source .txt. + try: + _MODEL.save_word2vec_format(str(bin_cache), binary=True) + except OSError: + pass # Read-only volume is fine; we'll just pay the txt-parse cost again. + return _MODEL + + +def _variant_candidates(word: str) -> list[str]: + """Casing/segmentation candidates ordered by specificity. + + PhoW2V-word uses underscores for compounds; PhoW2V-syllable has no + multi-token entries. Trying both forms covers either config without + caller branching. + """ + stripped = word.strip() + lowered = stripped.lower() + joined = stripped.replace(" ", "_") + joined_lower = lowered.replace(" ", "_") + # Ordered, de-duplicated. + seen: set[str] = set() + out: list[str] = [] + for c in (stripped, lowered, joined, joined_lower): + if c and c not in seen: + seen.add(c) + out.append(c) + return out + + +def canonicalize(kv: KeyedVectors, word: str) -> Optional[str]: + """Resolve `word` to its in-vocab form, trying exact → lower → space→underscore.""" + for candidate in _variant_candidates(word): + if candidate in kv: + return candidate + return None + + +def similarity(kv: KeyedVectors, a: str, b: str) -> float: + """Cosine similarity between two in-vocab keys. Caller must canonicalize.""" + return float(kv.similarity(a, b)) + + +def neighbors(kv: KeyedVectors, word: str, topn: int) -> list[tuple[str, float]]: + """Top-N nearest-neighbor keys with cosine scores. Caller must canonicalize.""" + return [(w, float(s)) for w, s in kv.most_similar(word, topn=topn)] + + +def _is_vietnamese_wordlike(word: str) -> bool: + """Reject digits and punctuation; accept Latin letters, Vietnamese diacritics, `_`.""" + for ch in word: + if ch == "_": + continue + cat = unicodedata.category(ch) + # Ll/Lu = letters, Mn = combining marks (diacritics on decomposed input). + if cat not in ("Ll", "Lu", "Lo", "Lt", "Mn"): + return False + return True + + +def random_word( + kv: KeyedVectors, + *, + min_rank: int = 0, + max_rank: Optional[int] = None, + alpha_only: bool = True, + min_len: int = 1, + max_len: int = 64, + max_attempts: int = 1000, +) -> Optional[str]: + """Return a random vocab key matching filters, or None within attempt budget. + + `index_to_key` is frequency-ordered for word2vec-text files, so rank bounds + behave as a frequency window. `alpha_only=True` accepts Vietnamese letters + and the word-boundary `_` — rejects numerals, punctuation, and foreign + scripts that sometimes leak into Vietnamese corpora. + """ + vocab = kv.index_to_key + upper = min(max_rank, len(vocab)) if max_rank is not None else len(vocab) + if min_rank >= upper: + return None + for _ in range(max_attempts): + word = vocab[_random.randrange(min_rank, upper)] + if not (min_len <= len(word) <= max_len): + continue + if alpha_only and not _is_vietnamese_wordlike(word): + continue + return word + return None diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7ecf4b0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +services: + phow2sim: + build: . + ports: + - "8001:8000" + environment: + # Variant: "word" (multi-syllable compounds, ~1.2GB) or "syllable" (single-syllable, ~150MB). + # To switch to syllables override all three: + # MODEL_URL: https://public.vinai.io/word2vec_vi_syllables_300dims.zip + # MODEL_PATH: /data/phow2v/word2vec_vi_syllables_300dims.txt + # MODEL_VARIANT: syllable + MODEL_URL: ${MODEL_URL:-https://public.vinai.io/word2vec_vi_words_300dims.zip} + MODEL_PATH: ${MODEL_PATH:-/data/phow2v/word2vec_vi_words_300dims.txt} + MODEL_VARIANT: ${MODEL_VARIANT:-word} + volumes: + - phow2v-cache:/data/phow2v + restart: unless-stopped + +volumes: + phow2v-cache: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e010326 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.115.4 +uvicorn[standard]==0.32.0 +gensim==4.3.3 +numpy<2 diff --git a/scripts/download-phow2v.sh b/scripts/download-phow2v.sh new file mode 100755 index 0000000..084eb72 --- /dev/null +++ b/scripts/download-phow2v.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Download a PhoW2V variant from VinAI's public mirror into ./models/. +# Usage: ./scripts/download-phow2v.sh [word|syllable] [100|300] +# Defaults: word, 300. +set -euo pipefail + +VARIANT="${1:-word}" +DIMS="${2:-300}" + +case "$VARIANT" in + word) SUFFIX="words" ;; + syllable) SUFFIX="syllables" ;; + *) echo "variant must be 'word' or 'syllable'" >&2; exit 2 ;; +esac + +case "$DIMS" in + 100|300) ;; + *) echo "dims must be 100 or 300" >&2; exit 2 ;; +esac + +URL="https://public.vinai.io/word2vec_vi_${SUFFIX}_${DIMS}dims.zip" +OUT_DIR="models" +ZIP_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.zip" +TXT_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.txt" + +mkdir -p "$OUT_DIR" + +if [[ -f "$TXT_PATH" ]]; then + echo "already present: $TXT_PATH" + exit 0 +fi + +echo "downloading $URL" +curl -fL --progress-bar -o "$ZIP_PATH" "$URL" + +echo "extracting to $OUT_DIR" +unzip -o -j "$ZIP_PATH" -d "$OUT_DIR" +rm -f "$ZIP_PATH" + +# Unzip may produce a differently-named .txt depending on archive contents. +# Rename to the expected path if a single .txt was extracted. +if [[ ! -f "$TXT_PATH" ]]; then + shopt -s nullglob + candidates=("$OUT_DIR"/*.txt) + if [[ ${#candidates[@]} -eq 1 ]]; then + mv "${candidates[0]}" "$TXT_PATH" + else + echo "warning: could not resolve extracted .txt path; check $OUT_DIR" >&2 + fi +fi + +echo "ready: $TXT_PATH"