From 6b1b401283a8a070cf0081136439d4f3bd40035f Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Thu, 23 Apr 2026 10:44:33 +0700 Subject: [PATCH] feat: fetch model via Nextcloud WebDAV with Basic auth The upstream public.vinai.io mirror is dead and PhoW2V's research license forbids public redistribution, so anonymous auto-download is no longer viable. Expect a private Nextcloud (WebDAV or password- protected public share) per deployment. - Stream downloads in 1MiB chunks (flat RAM for ~1GB zips) - Basic auth via MODEL_DOWNLOAD_USER / MODEL_DOWNLOAD_PASSWORD - Drop the broken public.vinai.io default; compose requires MODEL_URL - Add .env.example with WebDAV and public-share recipes - Remove scripts/download-phow2v.sh (pointed at the dead mirror) - README rewritten around the NC workflow; update license caveat --- .env.example | 15 ++++++ .gitignore | 2 + Dockerfile | 11 +++-- README.md | 94 ++++++++++++++++++++++++++------------ app/vectors.py | 37 +++++++++++++-- docker-compose.yml | 18 +++++--- scripts/download-phow2v.sh | 52 --------------------- 7 files changed, 134 insertions(+), 95 deletions(-) create mode 100644 .env.example delete mode 100755 scripts/download-phow2v.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d100b26 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Copy to `.env` and fill in your Nextcloud details. `.env` is gitignored. + +# --- WebDAV (recommended) --- +MODEL_URL=https://cloud.example.com/remote.php/dav/files/yourname/phow2v/word2vec_vi_words_300dims.zip +MODEL_DOWNLOAD_USER=yourname +MODEL_DOWNLOAD_PASSWORD=xxxx-xxxx-xxxx-xxxx-xxxx # Nextcloud app password + +# --- OR: password-protected public share --- +# MODEL_URL=https://cloud.example.com/s/abc123XYZ/download +# MODEL_DOWNLOAD_USER= +# MODEL_DOWNLOAD_PASSWORD=your-share-password + +# --- Variant selection (must match the file you uploaded) --- +MODEL_PATH=/data/phow2v/word2vec_vi_words_300dims.txt +MODEL_VARIANT=word diff --git a/.gitignore b/.gitignore index bf5bca3..8debfbb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ venv/ models/ *.zip *.txt.bin +plans/ +docs/ diff --git a/Dockerfile b/Dockerfile index 2084e1b..fffb065 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,15 +11,18 @@ RUN pip install --no-cache-dir -r requirements.txt COPY app ./app -# Defaults point at PhoW2V word-300d from VinAI's public mirror. -# Override MODEL_URL/MODEL_PATH to switch variants (syllables, 100d). -ENV MODEL_URL=https://public.vinai.io/word2vec_vi_words_300dims.zip \ - MODEL_PATH=/data/phow2v/word2vec_vi_words_300dims.txt \ +# MODEL_URL + MODEL_PATH + credentials are injected at runtime via +# docker-compose env/.env (see docker-compose.yml). No defaults here — +# PhoW2V's license forbids public redistribution, so every deployment +# must point at its own private mirror (typically Nextcloud WebDAV). +ENV MODEL_PATH=/data/phow2v/word2vec_vi_words_300dims.txt \ MODEL_VARIANT=word \ PORT=8000 EXPOSE 8000 +# First boot downloads ~1.2GB then parses ~60s; later boots use the +# cached .bin and only need ~10s. start-period accommodates both. HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=3 \ CMD curl -fsS http://localhost:8000/health || exit 1 diff --git a/README.md b/README.md index 7ff348f..2d3f5d7 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,17 @@ Tiny HTTP service that returns Vietnamese word2vec similarity and nearest neighbors — Vietnamese sibling of [`word2sim`](../word2sim). Same endpoint shapes; swap URLs and it's a drop-in replacement. -Backed by [**PhoW2V**](https://github.com/VinAIResearch/PhoW2V) (VinAI), the -largest pretrained Vietnamese word vectors available. Chosen over PhoBERT -for this purpose because word2vec's similarity distribution is wide -enough to drive a Semantle-style warmth meter, whereas raw transformer -embeddings saturate at the top. +Backed by [**PhoW2V**](https://github.com/datquocnguyen/PhoW2V) (VinAI / +Dat Quoc Nguyen), the largest pretrained Vietnamese word vectors +available. Chosen over PhoBERT for this purpose because word2vec's +similarity distribution is wide enough to drive a Semantle-style warmth +meter, whereas raw transformer embeddings saturate at the top. + +> **License note.** PhoW2V's research-only license forbids public +> redistribution, so this service doesn't — and can't — embed or +> auto-download the vectors from any public URL. You supply your own +> private mirror (typically a Nextcloud instance you control). See +> [Quick start](#quick-start). ## Stack @@ -63,45 +69,74 @@ tries exact → lowercase → space-to-underscore variants. ## Quick start -```bash -docker compose up --build -# First boot downloads ~1.2GB (word-300d) into the `phow2v-cache` volume. -# Model parse ~60s. A binary cache is written on first success so later -# restarts take ~10s. +1. **Get the vectors once.** Download from the [upstream Google Drive + mirror](https://drive.google.com/drive/folders/1NZhZFYbcwKzLpvvGdJUdPbwEVdVW4E3j?usp=drive_link) + (the one linked from the PhoW2V README — the original + `public.vinai.io` URLs are dead). You'll get four zips; keep the one + matching your chosen variant. + +2. **Upload the zip to your Nextcloud** in a folder like `phow2v/`. In + Nextcloud → Settings → Security, generate an **app password** for + this service (do not use your real login password, and do not disable + 2FA if you use it). + +3. **Configure env.** Copy `.env.example` to `.env` and fill in: + ```bash + cp .env.example .env + # then edit .env: + # MODEL_URL=https://cloud.example.com/remote.php/dav/files//phow2v/word2vec_vi_words_300dims.zip + # MODEL_DOWNLOAD_USER= + # MODEL_DOWNLOAD_PASSWORD= + ``` + +4. **Boot.** + ```bash + docker compose up --build + ``` + First boot streams ~1.2GB (word-300d) from Nextcloud into the + `phow2v-cache` volume, then parses ~60s. A binary `.bin` is written + alongside so later restarts load in ~10s. Health check start period + is 10 min to cover the first-boot cost. + +### Using a public share instead of WebDAV + +If you'd rather create a password-protected Nextcloud share link: + +``` +MODEL_URL=https://cloud.example.com/s//download +MODEL_DOWNLOAD_USER= +MODEL_DOWNLOAD_PASSWORD= ``` -Health check start period is 10 min to cover the download + parse. +Basic auth with an empty username is how Nextcloud authenticates a +public-share password. ## Switching variant -Edit `docker-compose.yml` or pass env: +Upload the desired zip to Nextcloud, then update `.env`: ```bash -MODEL_URL=https://public.vinai.io/word2vec_vi_syllables_100dims.zip \ -MODEL_PATH=/data/phow2v/word2vec_vi_syllables_100dims.txt \ -MODEL_VARIANT=syllable \ -docker compose up --build +MODEL_URL=https://cloud.example.com/remote.php/dav/files//phow2v/word2vec_vi_syllables_100dims.zip +MODEL_PATH=/data/phow2v/word2vec_vi_syllables_100dims.txt +MODEL_VARIANT=syllable ``` Delete the `phow2v-cache` volume when switching, otherwise the stale -`.bin` from the previous variant will load instead. - -## Manual model population - -Skip the auto-download if you want to prepare the volume ahead of time: +`.bin` from the previous variant will load instead: ```bash -./scripts/download-phow2v.sh word 300 # word-300d into ./models -# then mount ./models as /data/phow2v in docker-compose.yml +docker compose down -v && docker compose up --build ``` ## Config (env vars) | Var | Default | Meaning | |---|---|---| -| `MODEL_URL` | `https://public.vinai.io/word2vec_vi_words_300dims.zip` | fetched on first boot if `MODEL_PATH` absent | -| `MODEL_PATH` | `/data/phow2v/word2vec_vi_words_300dims.txt` | where the text-format vectors live | -| `MODEL_VARIANT` | `word` | declarative hint for the caller; `word` or `syllable` | +| `MODEL_URL` | *(required)* | Private URL to the PhoW2V zip. WebDAV or Nextcloud public-share `/download` URL. | +| `MODEL_DOWNLOAD_USER` | `""` | Basic-auth user. Empty for Nextcloud public-share password auth. | +| `MODEL_DOWNLOAD_PASSWORD` | *(required)* | Basic-auth password — Nextcloud app password, or share password. | +| `MODEL_PATH` | `/data/phow2v/word2vec_vi_words_300dims.txt` | Where the text-format vectors are persisted. | +| `MODEL_VARIANT` | `word` | `word` or `syllable`. Declarative hint; must match the file you uploaded. | ## Using from doantu (miti99bot) @@ -125,14 +160,13 @@ phow2sim/ ├── app/ │ ├── main.py # FastAPI routes │ └── vectors.py # PhoW2V loader + canonicalize + similarity/neighbors/random -├── scripts/ -│ └── download-phow2v.sh ├── Dockerfile ├── docker-compose.yml -└── requirements.txt +├── requirements.txt +└── .env.example # copy to .env and fill in Nextcloud creds ``` ## Credits -- Vectors: [PhoW2V](https://github.com/VinAIResearch/PhoW2V) by VinAI Research (research license — see their repo). +- Vectors: [PhoW2V](https://github.com/datquocnguyen/PhoW2V) by Dat Quoc Nguyen / VinAI Research (research-only license — see upstream). - API shape: sibling of [`word2sim`](../word2sim). diff --git a/app/vectors.py b/app/vectors.py index 75a4c9c..74226a3 100644 --- a/app/vectors.py +++ b/app/vectors.py @@ -9,10 +9,16 @@ use the fast binary path. Tokenization matters. The "word" variant expects underscore-joined compounds ("sinh_viên"); the "syllable" variant expects single syllables ("sinh", "viên"). Callers must normalize to match before querying. + +Model source: PhoW2V's research license forbids public redistribution, +so MODEL_URL is expected to point at a private, auth-gated mirror +(typically Nextcloud WebDAV). HTTP Basic auth is applied when +MODEL_DOWNLOAD_USER / MODEL_DOWNLOAD_PASSWORD are set. """ from __future__ import annotations +import base64 import os import random as _random import unicodedata @@ -24,20 +30,45 @@ from typing import Optional from gensim.models import KeyedVectors _MODEL: Optional[KeyedVectors] = None +_DOWNLOAD_CHUNK = 1 << 20 # 1 MiB; keeps peak RAM flat for ~1GB downloads. + + +def _build_request(url: str) -> urllib.request.Request: + """Build a GET with optional Basic auth from env. WebDAV-friendly.""" + req = urllib.request.Request(url) + user = os.environ.get("MODEL_DOWNLOAD_USER", "") + password = os.environ.get("MODEL_DOWNLOAD_PASSWORD") + # Password-only (empty user) covers Nextcloud public-share passwords too. + if password is not None: + creds = base64.b64encode(f"{user}:{password}".encode()).decode("ascii") + req.add_header("Authorization", f"Basic {creds}") + return req def _download_and_extract(url: str, target_txt: Path) -> None: - """Fetch a PhoW2V zip and extract its .txt into target_txt.""" + """Fetch a PhoW2V zip (streamed) and extract its .txt into target_txt.""" target_txt.parent.mkdir(parents=True, exist_ok=True) zip_path = target_txt.with_suffix(".zip") - urllib.request.urlretrieve(url, zip_path) + + req = _build_request(url) + with urllib.request.urlopen(req) as resp, open(zip_path, "wb") as dst: + while True: + chunk = resp.read(_DOWNLOAD_CHUNK) + if not chunk: + break + dst.write(chunk) + with zipfile.ZipFile(zip_path) as zf: txt_members = [m for m in zf.namelist() if m.endswith(".txt")] if not txt_members: raise RuntimeError(f"no .txt file inside {url}") # Flatten into target_txt regardless of archive's internal layout. with zf.open(txt_members[0]) as src, open(target_txt, "wb") as dst: - dst.write(src.read()) + while True: + chunk = src.read(_DOWNLOAD_CHUNK) + if not chunk: + break + dst.write(chunk) zip_path.unlink(missing_ok=True) diff --git a/docker-compose.yml b/docker-compose.yml index 7ecf4b0..b91a43a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,14 +4,20 @@ services: ports: - "8001:8000" environment: - # Variant: "word" (multi-syllable compounds, ~1.2GB) or "syllable" (single-syllable, ~150MB). - # To switch to syllables override all three: - # MODEL_URL: https://public.vinai.io/word2vec_vi_syllables_300dims.zip - # MODEL_PATH: /data/phow2v/word2vec_vi_syllables_300dims.txt - # MODEL_VARIANT: syllable - MODEL_URL: ${MODEL_URL:-https://public.vinai.io/word2vec_vi_words_300dims.zip} + # MODEL_URL is a Nextcloud WebDAV (or public-share+password) URL to a + # PhoW2V .zip. Example WebDAV path: + # https://cloud.example.com/remote.php/dav/files//phow2v/word2vec_vi_words_300dims.zip + # Example public share (password-protected): + # https://cloud.example.com/s//download (leave MODEL_DOWNLOAD_USER empty) + MODEL_URL: ${MODEL_URL:?set MODEL_URL in .env to your Nextcloud zip} MODEL_PATH: ${MODEL_PATH:-/data/phow2v/word2vec_vi_words_300dims.txt} MODEL_VARIANT: ${MODEL_VARIANT:-word} + # Basic-auth for the MODEL_URL. For Nextcloud WebDAV use the Nextcloud + # username + an app-password (not your login password). For a + # password-protected public share, leave MODEL_DOWNLOAD_USER empty + # and put the share password in MODEL_DOWNLOAD_PASSWORD. + MODEL_DOWNLOAD_USER: ${MODEL_DOWNLOAD_USER:-} + MODEL_DOWNLOAD_PASSWORD: ${MODEL_DOWNLOAD_PASSWORD:?set MODEL_DOWNLOAD_PASSWORD in .env} volumes: - phow2v-cache:/data/phow2v restart: unless-stopped diff --git a/scripts/download-phow2v.sh b/scripts/download-phow2v.sh deleted file mode 100755 index 084eb72..0000000 --- a/scripts/download-phow2v.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -# Download a PhoW2V variant from VinAI's public mirror into ./models/. -# Usage: ./scripts/download-phow2v.sh [word|syllable] [100|300] -# Defaults: word, 300. -set -euo pipefail - -VARIANT="${1:-word}" -DIMS="${2:-300}" - -case "$VARIANT" in - word) SUFFIX="words" ;; - syllable) SUFFIX="syllables" ;; - *) echo "variant must be 'word' or 'syllable'" >&2; exit 2 ;; -esac - -case "$DIMS" in - 100|300) ;; - *) echo "dims must be 100 or 300" >&2; exit 2 ;; -esac - -URL="https://public.vinai.io/word2vec_vi_${SUFFIX}_${DIMS}dims.zip" -OUT_DIR="models" -ZIP_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.zip" -TXT_PATH="${OUT_DIR}/word2vec_vi_${SUFFIX}_${DIMS}dims.txt" - -mkdir -p "$OUT_DIR" - -if [[ -f "$TXT_PATH" ]]; then - echo "already present: $TXT_PATH" - exit 0 -fi - -echo "downloading $URL" -curl -fL --progress-bar -o "$ZIP_PATH" "$URL" - -echo "extracting to $OUT_DIR" -unzip -o -j "$ZIP_PATH" -d "$OUT_DIR" -rm -f "$ZIP_PATH" - -# Unzip may produce a differently-named .txt depending on archive contents. -# Rename to the expected path if a single .txt was extracted. -if [[ ! -f "$TXT_PATH" ]]; then - shopt -s nullglob - candidates=("$OUT_DIR"/*.txt) - if [[ ${#candidates[@]} -eq 1 ]]; then - mv "${candidates[0]}" "$TXT_PATH" - else - echo "warning: could not resolve extracted .txt path; check $OUT_DIR" >&2 - fi -fi - -echo "ready: $TXT_PATH"