mirror of
https://github.com/tiennm99/phow2sim.git
synced 2026-09-03 02:20:18 +00:00
feat: initial phow2sim service
Tiny FastAPI service over PhoW2V Vietnamese word vectors. Mirrors word2sim's endpoint shapes (/similarity /neighbors /vocab /random) so clients can swap URLs without code changes. - Auto-downloads VinAI's PhoW2V on first boot, caches binary .bin for ~5x faster restarts - Viet-aware canonicalizer: exact -> lowercase -> space-to-underscore - Supports both word (compound) and syllable variants via env - Unicode-aware random-word filter accepts diacritics, rejects digits/punct
This commit is contained in:
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download a PhoW2V variant from VinAI's public mirror into ./models/.
|
||||
# Usage: ./scripts/download-phow2v.sh [word|syllable] [100|300]
|
||||
# Defaults: word, 300.
|
||||
set -euo pipefail
|
||||
|
||||
VARIANT="${1:-word}"
|
||||
DIMS="${2:-300}"
|
||||
|
||||
case "$VARIANT" in
|
||||
word) SUFFIX="words" ;;
|
||||
syllable) SUFFIX="syllables" ;;
|
||||
*) echo "variant must be 'word' or 'syllable'" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
case "$DIMS" in
|
||||
100|300) ;;
|
||||
*) echo "dims must be 100 or 300" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
URL="https://public.vinai.io/word2vec_vi_${SUFFIX}_${DIMS}dims.zip"
|
||||
OUT_DIR="models"
|
||||
ZIP_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.zip"
|
||||
TXT_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.txt"
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
if [[ -f "$TXT_PATH" ]]; then
|
||||
echo "already present: $TXT_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "downloading $URL"
|
||||
curl -fL --progress-bar -o "$ZIP_PATH" "$URL"
|
||||
|
||||
echo "extracting to $OUT_DIR"
|
||||
unzip -o -j "$ZIP_PATH" -d "$OUT_DIR"
|
||||
rm -f "$ZIP_PATH"
|
||||
|
||||
# Unzip may produce a differently-named .txt depending on archive contents.
|
||||
# Rename to the expected path if a single .txt was extracted.
|
||||
if [[ ! -f "$TXT_PATH" ]]; then
|
||||
shopt -s nullglob
|
||||
candidates=("$OUT_DIR"/*.txt)
|
||||
if [[ ${#candidates[@]} -eq 1 ]]; then
|
||||
mv "${candidates[0]}" "$TXT_PATH"
|
||||
else
|
||||
echo "warning: could not resolve extracted .txt path; check $OUT_DIR" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "ready: $TXT_PATH"
|
||||
Reference in New Issue
Block a user