feat: initial phow2sim service

Tiny FastAPI service over PhoW2V Vietnamese word vectors. Mirrors
word2sim's endpoint shapes (/similarity /neighbors /vocab /random) so
clients can swap URLs without code changes.

- Auto-downloads VinAI's PhoW2V on first boot, caches binary .bin for ~5x faster restarts
- Viet-aware canonicalizer: exact -> lowercase -> space-to-underscore
- Supports both word (compound) and syllable variants via env
- Unicode-aware random-word filter accepts diacritics, rejects digits/punct
This commit is contained in:
2026-04-23 10:05:50 +07:00
commit 8dd17acd4f
9 changed files with 553 additions and 0 deletions
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
# Download a PhoW2V variant from VinAI's public mirror into ./models/.
# Usage: ./scripts/download-phow2v.sh [word|syllable] [100|300]
# Defaults: word, 300.
set -euo pipefail
VARIANT="${1:-word}"
DIMS="${2:-300}"
case "$VARIANT" in
word) SUFFIX="words" ;;
syllable) SUFFIX="syllables" ;;
*) echo "variant must be 'word' or 'syllable'" >&2; exit 2 ;;
esac
case "$DIMS" in
100|300) ;;
*) echo "dims must be 100 or 300" >&2; exit 2 ;;
esac
URL="https://public.vinai.io/word2vec_vi_${SUFFIX}_${DIMS}dims.zip"
OUT_DIR="models"
ZIP_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.zip"
TXT_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.txt"
mkdir -p "$OUT_DIR"
if [[ -f "$TXT_PATH" ]]; then
echo "already present: $TXT_PATH"
exit 0
fi
echo "downloading $URL"
curl -fL --progress-bar -o "$ZIP_PATH" "$URL"
echo "extracting to $OUT_DIR"
unzip -o -j "$ZIP_PATH" -d "$OUT_DIR"
rm -f "$ZIP_PATH"
# Unzip may produce a differently-named .txt depending on archive contents.
# Rename to the expected path if a single .txt was extracted.
if [[ ! -f "$TXT_PATH" ]]; then
shopt -s nullglob
candidates=("$OUT_DIR"/*.txt)
if [[ ${#candidates[@]} -eq 1 ]]; then
mv "${candidates[0]}" "$TXT_PATH"
else
echo "warning: could not resolve extracted .txt path; check $OUT_DIR" >&2
fi
fi
echo "ready: $TXT_PATH"