diff --git a/.github/workflows/dev-beta-release.yaml b/.github/workflows/dev-beta-release.yaml index 5cbd83c2..6866df1b 100644 --- a/.github/workflows/dev-beta-release.yaml +++ b/.github/workflows/dev-beta-release.yaml @@ -340,6 +340,9 @@ jobs: GOCLAW_UPGRADE_TOKEN: ${{ secrets.ZUEY_GOCLAW_UPGRADE_TOKEN }} GOCLAW_DEPLOY_USER_ID: ${{ vars.ZUEY_GOCLAW_USER_ID || 'system' }} TAG: ${{ needs.beta_version.outputs.tag }} + ZUEY_SSH_HOST: ${{ vars.ZUEY_SSH_HOST || '82.197.71.246' }} + ZUEY_SSH_PORT: ${{ vars.ZUEY_SSH_PORT || '2233' }} + ZUEY_SSH_USER: ${{ vars.ZUEY_SSH_USER || 'zuey' }} steps: - name: Validate deploy configuration run: | @@ -352,6 +355,75 @@ jobs: done exit "$missing" + - name: Checkout repository (for VPS script sync) + uses: actions/checkout@v4 + with: + ref: ${{ needs.beta_version.outputs.tag }} + + - name: Sync zuey ops scripts to VPS + env: + ZUEY_SSH_PRIVATE_KEY: ${{ secrets.ZUEY_SSH_PRIVATE_KEY }} + ZUEY_SUDO_PASS: ${{ secrets.ZUEY_SUDO_PASS }} + run: | + set -euo pipefail + if [[ -z "${ZUEY_SSH_PRIVATE_KEY:-}" || -z "${ZUEY_SUDO_PASS:-}" ]]; then + echo "::warning::ZUEY_SSH_PRIVATE_KEY or ZUEY_SUDO_PASS not configured; skipping VPS script sync. Configure both repository secrets to keep /usr/local/bin/goclaw-deploy and /usr/local/bin/goclaw-upgrade-release in sync with the repo on every beta deploy." + exit 0 + fi + + # Verify the two scripts exist in the checked-out tag + for f in scripts/zuey/goclaw-deploy.sh scripts/zuey/goclaw-upgrade-release.sh; do + test -f "$f" || { echo "::error::$f missing in repo"; exit 1; } + bash -n "$f" || { echo "::error::$f has syntax error"; exit 1; } + done + + # Stage SSH key (BatchMode + StrictHostKeyChecking against known_hosts) + install -m 700 -d ~/.ssh + printf '%s\n' "$ZUEY_SSH_PRIVATE_KEY" > ~/.ssh/id_zuey + chmod 0600 ~/.ssh/id_zuey + ssh-keyscan -p "$ZUEY_SSH_PORT" -H "$ZUEY_SSH_HOST" >> ~/.ssh/known_hosts 2>/dev/null + + SSH_BASE=(-i ~/.ssh/id_zuey -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=15) + + # Upload (-O selects legacy scp protocol; OpenSSH 9.x defaults to sftp + # which is fine here, but -O is portable across runner image versions) + scp -O -P "$ZUEY_SSH_PORT" "${SSH_BASE[@]}" \ + scripts/zuey/goclaw-deploy.sh \ + scripts/zuey/goclaw-upgrade-release.sh \ + "${ZUEY_SSH_USER}@${ZUEY_SSH_HOST}:/tmp/" + + # Install on host: backup-if-changed → install (root:root 0755) → + # syntax check. The sudo password is shell-quoted via printf %q and + # interpolated into the remote command line; the SSH channel is + # encrypted and GitHub Actions auto-masks the secret in logs. + # sudo -S reads from stdin and does not echo the password. + quoted_pass=$(printf %q "$ZUEY_SUDO_PASS") + ssh -p "$ZUEY_SSH_PORT" "${SSH_BASE[@]}" "${ZUEY_SSH_USER}@${ZUEY_SSH_HOST}" \ + "SUDOPASS=$quoted_pass bash -s" <<'REMOTE' + set -euo pipefail + ts=$(date +%Y%m%d-%H%M%S) + for name in goclaw-deploy goclaw-upgrade-release; do + src="/tmp/${name}.sh" + dst="/usr/local/bin/${name}" + if [ ! -f "$src" ]; then echo "::error::missing $src"; exit 1; fi + if [ -f "$dst" ] && cmp -s "$src" "$dst"; then + echo "no change: $name" + rm -f "$src" + continue + fi + if [ -f "$dst" ]; then + echo "$SUDOPASS" | sudo -S cp -p "$dst" "${dst}.bak-${ts}" + fi + echo "$SUDOPASS" | sudo -S install -o root -g root -m 0755 "$src" "$dst" + echo "$SUDOPASS" | sudo -S bash -n "$dst" + rm -f "$src" + echo "installed: $name" + done + REMOTE + + # Clean up the private key from the runner FS + shred -u ~/.ssh/id_zuey 2>/dev/null || rm -f ~/.ssh/id_zuey + - name: Trigger zuey gateway upgrade run: | base_url="${GOCLAW_DEPLOY_URL%/}" diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 29bbd4f9..6ed63241 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -47,10 +47,10 @@ export GOCLAW_DOMAIN= | `/var/lib/goclaw/data` | GoClaw persistent data | | `/var/lib/goclaw/workspace` | Agent workspace | | `/var/lib/goclaw/postgres` | Postgres Docker data | -| `/usr/local/bin/goclaw-deploy` | Release switch, upgrade, health-check, rollback | +| `/usr/local/bin/goclaw-deploy` | Release switch, upgrade, health-check, rollback (source: [`scripts/zuey/goclaw-deploy.sh`](../scripts/zuey/goclaw-deploy.sh)) | | `/usr/local/bin/goclaw-issue-ssl` | Certbot wrapper for the deployment domain | | `/usr/local/bin/goclaw-backup-r2` | Postgres dump, R2 upload, retention cleanup | -| `/usr/local/bin/goclaw-upgrade-release` | Download and deploy a GitHub Release tarball | +| `/usr/local/bin/goclaw-upgrade-release` | Download and deploy a GitHub Release tarball (source: [`scripts/zuey/goclaw-upgrade-release.sh`](../scripts/zuey/goclaw-upgrade-release.sh)) | Secrets are stored only in server env files. Do not copy tokens or database passwords into repo docs. @@ -248,6 +248,68 @@ ssh -p "$GOCLAW_SSH_PORT" "$GOCLAW_SSH_USER@$GOCLAW_HOST" "chmod +x /opt/goclaw/ 5. Poll `/health`. 6. Roll back symlink and restart if health fails. +### Self-loop symlink guard + +`goclaw-deploy` captures the previous release for rollback via `readlink -f /opt/goclaw/current` before swinging the symlink. If `/opt/goclaw/current` is ever a self-loop (e.g. `current -> current`) or otherwise unresolvable, `readlink -f` exits non-zero. Combined with `set -euo pipefail`, an unguarded call aborts the script before `ln -sfn` runs, so deploys fail silently and zero rollback target is recorded — observed in production on 2026-05-27 where every `deploy_zuey_beta` CI run failed at this step. + +The script swallows the readlink failure (`readlink -f ... 2>/dev/null || true`) and logs a warning when the symlink exists but resolves to empty, then proceeds to overwrite. Rollback is skipped in that case because no valid `previous` is available. + +If you ever edit `/usr/local/bin/goclaw-deploy` on zuey, preserve this guard: + +```bash +previous="" +if [ -L /opt/goclaw/current ]; then previous="$(readlink -f /opt/goclaw/current 2>/dev/null || true)"; fi +if [ -z "$previous" ] && [ -L /opt/goclaw/current ]; then + echo "warn: /opt/goclaw/current is a symlink but readlink -f failed (likely self-loop or broken target); overwriting without rollback target" >&2 +fi +``` + +The canonical source of this script lives in the repo at [`scripts/zuey/goclaw-deploy.sh`](../scripts/zuey/goclaw-deploy.sh), alongside [`scripts/zuey/goclaw-upgrade-release.sh`](../scripts/zuey/goclaw-upgrade-release.sh). The VPS copies at `/usr/local/bin/goclaw-deploy` and `/usr/local/bin/goclaw-upgrade-release` are downstream replicas. CI auto-syncs both on every beta release via the `Sync zuey ops scripts to VPS` step in `.github/workflows/dev-beta-release.yaml`. For manual sync (off-CI): + +```bash +scp -P 2233 scripts/zuey/goclaw-deploy.sh scripts/zuey/goclaw-upgrade-release.sh \ + zuey@82.197.71.246:/tmp/ +ssh -p 2233 zuey@82.197.71.246 'bash -s' <<'EOF' +set -euo pipefail +ts=$(date +%Y%m%d-%H%M%S) +for name in goclaw-deploy goclaw-upgrade-release; do + if [ -f "/usr/local/bin/$name" ]; then + sudo cp -p "/usr/local/bin/$name" "/usr/local/bin/${name}.bak-${ts}" + fi + sudo install -o root -g root -m 0755 "/tmp/${name}.sh" "/usr/local/bin/$name" + sudo bash -n "/usr/local/bin/$name" && echo "OK: $name" +done +EOF +``` + +Always back up the live copy before overwriting (the `cp -p` line above does this). + +### CI auto-sync — required GitHub secrets + +The `Sync zuey ops scripts to VPS` step in `dev-beta-release.yaml` runs `scp + sudo install` before triggering the gateway upgrade endpoint. It needs the following repository secrets configured under **Settings → Secrets and variables → Actions**: + +| Secret | Purpose | +|---|---| +| `ZUEY_SSH_PRIVATE_KEY` | CI-only ed25519/rsa key; its public key must be appended to `zuey@82.197.71.246:~/.ssh/authorized_keys`. **Do not reuse the operator's personal key.** | +| `ZUEY_SUDO_PASS` | Same value as `ZUEY_GOCLAW_SUDO_PASS` in the operator's local `.env`; used by `sudo -S` over the SSH session to install scripts. | + +Optional repository **variables** (override defaults if the VPS endpoint changes): + +| Variable | Default | +|---|---| +| `ZUEY_SSH_HOST` | `82.197.71.246` | +| `ZUEY_SSH_PORT` | `2233` | +| `ZUEY_SSH_USER` | `zuey` | + +To rotate the CI SSH key: + +```bash +ssh-keygen -t ed25519 -f /tmp/ci-zuey-key -N '' -C 'gh-actions-deploy-zuey-beta' +# add /tmp/ci-zuey-key.pub to zuey:~/.ssh/authorized_keys (consider restricting to scp+install via `command="..."` forced-command) +# paste contents of /tmp/ci-zuey-key into the ZUEY_SSH_PRIVATE_KEY secret +# then `shred -u /tmp/ci-zuey-key*` locally +``` + ## Backup And Restore Automated backups: diff --git a/docs/project-changelog.md b/docs/project-changelog.md index c2d52201..9cc2b8b1 100644 --- a/docs/project-changelog.md +++ b/docs/project-changelog.md @@ -4,6 +4,23 @@ Significant changes, features, and fixes in reverse chronological order. --- +## 2026-05-27 + +### zuey VPS ops scripts: repo-tracked + CI auto-sync + +**Fixes** + +- Patched `/usr/local/bin/goclaw-deploy` on zuey to survive a self-loop `/opt/goclaw/current` symlink (`readlink -f` now `2>/dev/null || true`, with a warning when `previous` is empty). Without this, `set -euo pipefail` aborted before `ln -sfn` could overwrite the symlink, silently failing every `deploy_zuey_beta` CI run. + +**Changes** + +- Moved `scripts/goclaw-upgrade-release.sh` → `scripts/zuey/goclaw-upgrade-release.sh`. +- Added `scripts/zuey/goclaw-deploy.sh` (canonical source for the on-host `/usr/local/bin/goclaw-deploy`). +- Wired `Sync zuey ops scripts to VPS` step in `.github/workflows/dev-beta-release.yaml` to `scp + sudo install` both scripts before triggering the gateway upgrade endpoint on every beta release. Requires new repository secrets `ZUEY_SSH_PRIVATE_KEY` and `ZUEY_SUDO_PASS`; step skips with a warning if either is unset. +- Updated `docs/deployment-guide.md` with the self-loop guard rationale, manual sync recipe, and required-secrets table. + +--- + ## 2026-05-24 ### Google Workspace CLI runtime integration diff --git a/scripts/zuey/goclaw-deploy.sh b/scripts/zuey/goclaw-deploy.sh new file mode 100755 index 00000000..b3015506 --- /dev/null +++ b/scripts/zuey/goclaw-deploy.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Canonical source for /usr/local/bin/goclaw-deploy on the zuey VPS. +# +# This repo is the source of truth; the VPS copy is a downstream replica. +# Auto-synced on every beta release by the `Sync zuey ops scripts to VPS` +# step in `.github/workflows/dev-beta-release.yaml`. See +# `docs/deployment-guide.md` for the manual sync recipe and required +# GitHub secrets. +# +# Self-loop guard (2026-05-27 incident): when `/opt/goclaw/current` is a +# self-referential symlink (current -> current), `readlink -f` exits non-zero. +# With `set -euo pipefail`, that aborts the script before `ln -sfn` can fix +# the symlink — causing silent CI deploy failures. The guard below swallows +# the readlink failure and logs a warning when the symlink resolves to empty, +# so the deploy still completes (without a rollback target). +set -euo pipefail +release_dir="${1:?usage: goclaw-deploy /opt/goclaw/releases/}" +if [ ! -x "$release_dir/goclaw" ]; then echo "missing executable: $release_dir/goclaw" >&2; exit 2; fi +if [ ! -d "$release_dir/migrations" ]; then echo "missing migrations dir" >&2; exit 2; fi +previous="" +if [ -L /opt/goclaw/current ]; then previous="$(readlink -f /opt/goclaw/current 2>/dev/null || true)"; fi +if [ -z "$previous" ] && [ -L /opt/goclaw/current ]; then + echo "warn: /opt/goclaw/current is a symlink but readlink -f failed (likely self-loop or broken target); overwriting without rollback target" >&2 +fi +ln -sfn "$release_dir" /opt/goclaw/current +chown -h goclaw:goclaw /opt/goclaw/current || true +chown -R root:root "$release_dir" +chmod 755 "$release_dir/goclaw" +set -a +. /etc/goclaw/goclaw.env +set +a +systemctl daemon-reload +if systemctl is-active --quiet goclaw; then systemctl stop goclaw; fi +/opt/goclaw/current/goclaw upgrade +systemctl start goclaw +for i in $(seq 1 60); do + if curl -fsS http://127.0.0.1:18790/health >/tmp/goclaw-health.json; then + cat /tmp/goclaw-health.json + echo + exit 0 + fi + sleep 2 +done +echo "health check failed" >&2 +journalctl -u goclaw -n 120 --no-pager >&2 || true +if [ -n "$previous" ] && [ -d "$previous" ]; then + echo "rolling back to $previous" >&2 + ln -sfn "$previous" /opt/goclaw/current + systemctl restart goclaw || true +fi +exit 1 diff --git a/scripts/goclaw-upgrade-release.sh b/scripts/zuey/goclaw-upgrade-release.sh old mode 100644 new mode 100755 similarity index 100% rename from scripts/goclaw-upgrade-release.sh rename to scripts/zuey/goclaw-upgrade-release.sh