fix(cliproxy): harden update recovery paths

docker/host/ccs-cliproxy-reconcile.sh

@@ -3,8 +3,9 @@
 set -Eeuo pipefail
 
 container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
-compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker}"
-compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.integrated.yml}"
+compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/opt/cliproxy}"
+compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.yml}"
+compose_project="${CCS_CLIPROXY_COMPOSE_PROJECT:-docker}"
 lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
 log_file="${CCS_CLIPROXY_RECONCILE_LOG:-/var/log/ccs-cliproxy-reconcile.log}"
 
@@ -37,8 +38,10 @@ wait_for_health() {
 
 compose_up() {
   cd "$compose_dir"
-  docker compose -f "$compose_file" up -d --no-build || \
-    docker compose -f "$compose_file" up -d --build
+  docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
+    -f "$compose_file" up -d --no-build || \
+    docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
+      -f "$compose_file" up -d --build
 }
 
 if ! docker inspect "$container_name" >/dev/null 2>&1; then
@@ -67,10 +70,12 @@ if probe; then
 fi
 
 log 'Dashboard or proxy failed two consecutive probes; restarting supervised processes'
-docker exec "$container_name" supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy
-if wait_for_health; then
-  log 'Supervised processes recovered and passed both health probes'
-  exit 0
+if docker exec "$container_name" \
+  supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy; then
+  if wait_for_health; then
+    log 'Supervised processes recovered and passed both health probes'
+    exit 0
+  fi
 fi
 
 log 'Supervisor recovery failed; restarting the container'

docker/host/ccs-cliproxy-safe-update.sh

@@ -26,11 +26,16 @@ if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'tru
   exit 1
 fi
 
-status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)" || {
+status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus --verbose 2>&1)" || {
   log "Unable to inspect the installed CLIProxy version: $status_output"
   exit 1
 }
 
+if grep -qi 'Could not fetch' <<<"$status_output"; then
+  log "Unable to check the latest CLIProxy version: $(grep -im1 'Could not fetch' <<<"$status_output" | xargs)"
+  exit 1
+fi
+
 if ! grep -qi 'update available' <<<"$status_output"; then
   exit 0
 fi
@@ -50,7 +55,7 @@ stage_binary="$stage_dir/cli-proxy-api-plus"
 stage_version="$stage_dir/.version"
 backup_binary="$stage_root/previous-binary"
 backup_version="$stage_root/previous-version"
-swap_started=0
+maintenance_started=0
 
 supervisorctl_cmd() {
   supervisorctl -c /etc/supervisord.conf "$@"
@@ -73,26 +78,40 @@ wait_for_proxy() {
 }
 
 rollback() {
+  rollback_ok=1
   supervisorctl_cmd stop cliproxy >/dev/null 2>&1 || true
   if [ -f "$backup_binary" ]; then
-    install -m 0755 "$backup_binary" "$live_binary.rollback"
-    mv -f "$live_binary.rollback" "$live_binary"
+    install -m 0755 "$backup_binary" "$live_binary.rollback" && \
+      mv -f "$live_binary.rollback" "$live_binary" || rollback_ok=0
+  else
+    rollback_ok=0
   fi
   if [ -f "$backup_version" ]; then
-    install -m 0644 "$backup_version" "$live_version.rollback"
-    mv -f "$live_version.rollback" "$live_version"
+    install -m 0644 "$backup_version" "$live_version.rollback" && \
+      mv -f "$live_version.rollback" "$live_version" || rollback_ok=0
+  else
+    rollback_ok=0
   fi
-  supervisorctl_cmd start cliproxy >/dev/null
-  wait_for_proxy
+  supervisorctl_cmd start cliproxy >/dev/null || rollback_ok=0
+  wait_for_proxy || rollback_ok=0
+  [ "$rollback_ok" -eq 1 ]
 }
 
 on_exit() {
   rc=$?
   trap - EXIT INT TERM HUP
-  if [ "$rc" -ne 0 ] && [ "$swap_started" -eq 1 ]; then
-    rollback || true
+  rollback_failed=0
+  if [ "$rc" -ne 0 ] && [ "$maintenance_started" -eq 1 ]; then
+    if ! rollback; then
+      rollback_failed=1
+      printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2
+    fi
+  fi
+  if [ "$rollback_failed" -eq 0 ]; then
+    cleanup
+  else
+    rc=70
   fi
-  cleanup
   exit "$rc"
 }
 
@@ -107,17 +126,17 @@ test -s "$stage_version"
 cp -p "$live_binary" "$backup_binary"
 cp -p "$live_version" "$backup_version"
 
+maintenance_started=1
 supervisorctl_cmd stop cliproxy >/dev/null
-swap_started=1
 mv -f "$stage_binary" "$live_binary"
 mv -f "$stage_version" "$live_version"
 chmod 0755 "$live_binary"
 supervisorctl_cmd start cliproxy >/dev/null
 wait_for_proxy
-swap_started=0
+maintenance_started=0
 CONTAINER_UPDATE
 then
-  log 'CLIProxy Plus update failed; previous binary was restored and restarted'
+  log 'CLIProxy Plus update failed; rollback was attempted and reconciliation will verify service health'
   exit 1
 fi
 

docker/host/systemd/ccs-cliproxy-reconcile.service

@@ -6,4 +6,4 @@ Requires=docker.service
 [Service]
 Type=oneshot
 ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh
-TimeoutStartSec=5min
+TimeoutStartSec=10min

src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts

@@ -4,6 +4,7 @@ import * as os from 'os';
 import * as path from 'path';
 import { getExecutableName } from '../platform-detector';
 import { downloadAndInstall } from '../installer';
+import { ensureBinary } from '../lifecycle';
 
 describe('atomic binary installation', () => {
   let binPath: string;
@@ -148,4 +149,162 @@ describe('atomic binary installation', () => {
     expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
     expect(stagingDirectories()).toEqual([]);
   });
+
+  it('installs a forced version even when an older binary already exists', async () => {
+    const binaryPath = path.join(binPath, getExecutableName('original'));
+    fs.writeFileSync(binaryPath, 'old-binary');
+    let installs = 0;
+
+    const resolvedPath = await ensureBinary(
+      {
+        version: '6.7.1',
+        releaseUrl: 'https://example.invalid',
+        binPath,
+        maxRetries: 1,
+        verbose: false,
+        forceVersion: true,
+        skipAutoUpdate: false,
+        allowInstall: true,
+        backend: 'original',
+      },
+      {
+        downloadAndInstallFn: async () => {
+          installs += 1;
+        },
+      }
+    );
+
+    expect(resolvedPath).toBe(binaryPath);
+    expect(installs).toBe(1);
+  });
+
+  it('restores the previous binary when publishing the version marker fails', async () => {
+    const binaryName = getExecutableName('original');
+    const binaryPath = path.join(binPath, binaryName);
+    const versionPath = path.join(binPath, '.version');
+    fs.writeFileSync(binaryPath, 'old-binary');
+    fs.writeFileSync(versionPath, '6.6.80');
+    let renames = 0;
+
+    await expect(
+      downloadAndInstall(
+        {
+          version: '6.7.1',
+          releaseUrl: 'https://example.invalid',
+          binPath,
+          maxRetries: 1,
+          verbose: false,
+          forceVersion: true,
+          skipAutoUpdate: false,
+          allowInstall: true,
+          backend: 'original',
+        },
+        false,
+        {
+          downloadWithRetryFn: async (_url, archivePath) => {
+            fs.writeFileSync(archivePath, 'downloaded-archive');
+            return { success: true, filePath: archivePath, retries: 0 };
+          },
+          verifyChecksumFn: async () => ({
+            valid: true,
+            expected: 'checksum',
+            actual: 'checksum',
+          }),
+          extractArchiveFn: async (_archivePath, destination) => {
+            fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
+          },
+          renameSyncFn: (source, destination) => {
+            renames += 1;
+            if (renames === 2) throw new Error('version marker blocked');
+            fs.renameSync(source, destination);
+          },
+        }
+      )
+    ).rejects.toThrow('version marker blocked');
+
+    expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
+    expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
+    expect(stagingDirectories()).toEqual([]);
+  });
+
+  it('serializes concurrent installs so the binary and version stay paired', async () => {
+    const binaryName = getExecutableName('original');
+    const binaryPath = path.join(binPath, binaryName);
+    const versionPath = path.join(binPath, '.version');
+
+    const install = (version: string) =>
+      downloadAndInstall(
+        {
+          version,
+          releaseUrl: 'https://example.invalid',
+          binPath,
+          maxRetries: 1,
+          verbose: false,
+          forceVersion: true,
+          skipAutoUpdate: false,
+          allowInstall: true,
+          backend: 'original',
+        },
+        false,
+        {
+          downloadWithRetryFn: async (_url, archivePath) => {
+            fs.writeFileSync(archivePath, 'downloaded-archive');
+            return { success: true, filePath: archivePath, retries: 0 };
+          },
+          verifyChecksumFn: async () => ({
+            valid: true,
+            expected: 'checksum',
+            actual: 'checksum',
+          }),
+          extractArchiveFn: async (_archivePath, destination) => {
+            fs.writeFileSync(path.join(destination, binaryName), `binary-${version}`);
+          },
+        }
+      );
+
+    await Promise.all([install('6.7.1'), install('6.7.2')]);
+
+    const installedVersion = fs.readFileSync(versionPath, 'utf8');
+    expect(fs.readFileSync(binaryPath, 'utf8')).toBe(`binary-${installedVersion}`);
+    expect(stagingDirectories()).toEqual([]);
+  });
+
+  it('removes staging residue left by an interrupted prior install', async () => {
+    const stalePath = path.join(binPath, '.cliproxy-install-stale');
+    fs.mkdirSync(stalePath);
+    fs.writeFileSync(path.join(stalePath, 'partial-archive'), 'partial');
+    const binaryName = getExecutableName('original');
+
+    await downloadAndInstall(
+      {
+        version: '6.7.1',
+        releaseUrl: 'https://example.invalid',
+        binPath,
+        maxRetries: 1,
+        verbose: false,
+        forceVersion: true,
+        skipAutoUpdate: false,
+        allowInstall: true,
+        backend: 'original',
+      },
+      false,
+      {
+        downloadWithRetryFn: async (_url, archivePath) => {
+          fs.writeFileSync(archivePath, 'downloaded-archive');
+          return { success: true, filePath: archivePath, retries: 0 };
+        },
+        verifyChecksumFn: async () => ({
+          valid: true,
+          expected: 'checksum',
+          actual: 'checksum',
+        }),
+        extractArchiveFn: async (_archivePath, destination) => {
+          fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
+        },
+      }
+    );
+
+    expect(fs.existsSync(stalePath)).toBe(false);
+    expect(stagingDirectories()).toEqual([]);
+  });
 });

src/cliproxy/binary/installer.ts

@@ -5,6 +5,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
+import * as lockfile from 'proper-lockfile';
 import { BinaryManagerConfig } from '../types';
 import {
   detectPlatform,
@@ -16,7 +17,6 @@ import {
 import { downloadWithRetry } from './downloader';
 import { verifyChecksum, computeChecksum } from './verifier';
 import { extractArchive } from './extractor';
-import { writeInstalledVersion } from './version-cache';
 import { ProgressIndicator } from '../../utils/progress-indicator';
 import { ok } from '../../utils/ui';
 import { BinaryError, NetworkError } from '../../errors/error-types';
@@ -47,14 +47,29 @@ export async function downloadAndInstall(
   const renameSyncFn = deps.renameSyncFn ?? fs.renameSync;
 
   fs.mkdirSync(config.binPath, { recursive: true });
-  const stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-'));
-  const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`);
-  const stagedBinary = path.join(stagingPath, getExecutableName(backend));
-  const installedBinary = path.join(config.binPath, getExecutableName(backend));
+  const releaseLock = await lockfile.lock(config.binPath, {
+    stale: 10 * 60 * 1000,
+    retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 },
+  });
+  let stagingPath: string | undefined;
   const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`);
-  spinner.start();
 
   try {
+    for (const entry of fs.readdirSync(config.binPath)) {
+      if (entry.startsWith('.cliproxy-install-')) {
+        fs.rmSync(path.join(config.binPath, entry), { recursive: true, force: true });
+      }
+    }
+    stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-'));
+    const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`);
+    const stagedBinary = path.join(stagingPath, getExecutableName(backend));
+    const stagedVersion = path.join(stagingPath, '.version');
+    const installedBinary = path.join(config.binPath, getExecutableName(backend));
+    const installedVersion = path.join(config.binPath, '.version');
+    const backupBinary = path.join(stagingPath, '.previous-binary');
+    const hadInstalledBinary = fs.existsSync(installedBinary);
+    spinner.start();
+
     const result = await downloadWithRetryFn(downloadUrl, archivePath, {
       maxRetries: config.maxRetries,
       verbose,
@@ -95,15 +110,31 @@ export async function downloadAndInstall(
       if (verbose) console.error(`[cliproxy] Set executable permissions: ${stagedBinary}`);
     }
 
+    fs.writeFileSync(stagedVersion, config.version, 'utf8');
+    if (hadInstalledBinary) {
+      fs.copyFileSync(installedBinary, backupBinary);
+      if (platform.os !== 'windows') fs.chmodSync(backupBinary, 0o755);
+    }
+
     renameSyncFn(stagedBinary, installedBinary);
-    writeInstalledVersion(config.binPath, config.version);
+    try {
+      renameSyncFn(stagedVersion, installedVersion);
+    } catch (error) {
+      if (hadInstalledBinary) {
+        renameSyncFn(backupBinary, installedBinary);
+      } else {
+        fs.unlinkSync(installedBinary);
+      }
+      throw error;
+    }
     spinner.succeed(`${backendLabel} ready`);
     console.log(ok(`${backendLabel} v${config.version} installed successfully`));
   } catch (error) {
     spinner.fail('Installation failed');
     throw error;
   } finally {
-    fs.rmSync(stagingPath, { recursive: true, force: true });
+    if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
+    await releaseLock();
   }
 }
 

src/cliproxy/binary/lifecycle.ts

@@ -15,6 +15,7 @@ import { downloadAndInstall, getBinaryPath } from './installer';
 import { info, warn } from '../../utils/ui';
 import { isCliproxyRunning } from '../services/stats-fetcher';
 import { resolveLifecyclePort } from '../config/port-manager';
+import { BinaryError } from '../../errors/error-types';
 import {
   CLIPROXY_MAX_STABLE_VERSION,
   CLIPROXY_FAULTY_RANGE,
@@ -103,17 +104,26 @@ async function handleAutoUpdate(config: BinaryManagerConfig, verbose: boolean):
  * Ensure binary is available (download if missing, update if outdated)
  * @returns Path to executable binary
  */
-export async function ensureBinary(config: BinaryManagerConfig): Promise<string> {
+interface EnsureBinaryDeps {
+  downloadAndInstallFn?: typeof downloadAndInstall;
+}
+
+export async function ensureBinary(
+  config: BinaryManagerConfig,
+  deps: EnsureBinaryDeps = {}
+): Promise<string> {
   const verbose = config.verbose;
   const backend: CLIProxyBackend = config.backend ?? DEFAULT_BACKEND;
   const binaryPath = getBinaryPath(config.binPath, backend);
+  const downloadAndInstallFn = deps.downloadAndInstallFn ?? downloadAndInstall;
 
   // Binary exists - check for updates unless forceVersion
   if (fs.existsSync(binaryPath)) {
     log(`Binary exists: ${binaryPath}`, verbose);
 
     if (config.forceVersion) {
-      log('Force version mode: skipping auto-update', verbose);
+      log(`Force version mode: installing specified version ${config.version}`, verbose);
+      await downloadAndInstallFn(config, verbose);
       return binaryPath;
     }
 
@@ -134,9 +144,10 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise<string>
 
   // Binary missing
   if (!config.allowInstall) {
-    throw new Error(
+    throw new BinaryError(
       `${getBackendLabel(backend)} binary is not installed locally. ` +
-        'Run "ccs cliproxy install" when you have network access.'
+        'Run "ccs cliproxy install" when you have network access.',
+      binaryPath
     );
   }
 
@@ -161,6 +172,6 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise<string>
     log(`Force version mode: using specified version ${config.version}`, verbose);
   }
 
-  await downloadAndInstall(config, verbose);
+  await downloadAndInstallFn(config, verbose);
   return binaryPath;
 }

src/web-server/services/cliproxy-dashboard-install-service.ts

@@ -4,6 +4,7 @@ import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/s
 import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker';
 import { isCliproxyRunning } from '../../cliproxy/services/stats-fetcher';
 import type { CLIProxyBackend } from '../../cliproxy/types';
+import { ProxyError } from '../../errors/error-types';
 import {
   isRunningUnderSupervisord,
   restartCliproxyViaSupervisord,
@@ -22,6 +23,8 @@ interface InstallDashboardCliproxyVersionDeps {
     backend?: CLIProxyBackend
   ) => Promise<void>;
   ensureCliproxyService: () => Promise<ServiceStartResult>;
+  isRunningUnderSupervisord?: () => boolean;
+  restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord;
 }
 
 const defaultDeps: InstallDashboardCliproxyVersionDeps = {
@@ -48,6 +51,23 @@ async function wasProxyRunning(deps: InstallDashboardCliproxyVersionDeps): Promi
   return deps.isCliproxyRunning();
 }
 
+async function restoreProxyService(
+  deps: InstallDashboardCliproxyVersionDeps
+): Promise<ServiceStartResult> {
+  const underSupervisord = deps.isRunningUnderSupervisord?.() ?? isRunningUnderSupervisord();
+  if (underSupervisord) {
+    const restart = deps.restartCliproxyViaSupervisord?.() ?? restartCliproxyViaSupervisord();
+    return {
+      started: restart.success,
+      alreadyRunning: false,
+      port: restart.port ?? resolveLifecyclePort(),
+      error: restart.error,
+    };
+  }
+
+  return deps.ensureCliproxyService();
+}
+
 export async function installDashboardCliproxyVersion(
   version: string,
   backend: CLIProxyBackend,
@@ -59,7 +79,21 @@ export async function installDashboardCliproxyVersion(
 
   // The installer owns the stop-and-replace lifecycle, including best-effort
   // shutdown for tracked and untracked proxies before swapping the binary.
-  await deps.installCliproxyVersion(version, true, effectiveBackend);
+  try {
+    await deps.installCliproxyVersion(version, true, effectiveBackend);
+  } catch (error) {
+    if (shouldRestoreService) {
+      const restoreResult = await restoreProxyService(deps);
+      if (!restoreResult.started && !restoreResult.alreadyRunning) {
+        const installMessage = error instanceof Error ? error.message : String(error);
+        throw new ProxyError(
+          `${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
+          restoreResult.port
+        );
+      }
+    }
+    throw error;
+  }
 
   if (!shouldRestoreService) {
     return {
@@ -70,20 +104,7 @@ export async function installDashboardCliproxyVersion(
   }
 
   // In Docker, supervisord owns process lifecycle — delegate restart to it
-  if (isRunningUnderSupervisord()) {
-    const result = restartCliproxyViaSupervisord();
-    return {
-      success: result.success,
-      restarted: result.success,
-      port: result.port,
-      error: result.error,
-      message: result.success
-        ? `Successfully installed ${backendLabel} v${version} and restarted it on port ${result.port}`
-        : `Installed ${backendLabel} v${version}, but restart failed`,
-    };
-  }
-
-  const startResult = await deps.ensureCliproxyService();
+  const startResult = await restoreProxyService(deps);
   if (!startResult.started && !startResult.alreadyRunning) {
     return {
       success: false,

tests/unit/docker/cliproxy-host-continuity-assets.test.ts

@@ -35,9 +35,15 @@ describe('CLIProxy Docker host continuity assets', () => {
   });
 
   it('recreates missing containers and escalates unhealthy recovery', () => {
-    expect(reconcileScript).toContain('docker compose -f "$compose_file" up -d --no-build');
+    expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_DIR:-/opt/cliproxy');
+    expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_PROJECT:-docker');
+    expect(reconcileScript).toContain('--project-name "$compose_project"');
+    expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build');
     expect(reconcileScript).toContain('restart ccs-dashboard cliproxy');
     expect(reconcileScript).toContain('docker restart "$container_name"');
+    expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan(
+      reconcileScript.indexOf('restart ccs-dashboard cliproxy')
+    );
   });
 
   it('installs executable services on bounded timers', () => {

tests/unit/web-server/cliproxy-dashboard-install-service.test.ts

@@ -10,6 +10,7 @@ function createDeps(
     sessionRunning?: boolean;
     remoteRunning?: boolean;
     startResult?: { started: boolean; alreadyRunning: boolean; port: number; error?: string };
+    installError?: Error;
   } = {}
 ) {
   const calls = {
@@ -30,6 +31,7 @@ function createDeps(
       _backend?: CLIProxyBackend
     ) => {
       calls.installCliproxyVersion += 1;
+      if (overrides.installError) throw overrides.installError;
     },
     ensureCliproxyService: async () => {
       calls.ensureCliproxyService += 1;
@@ -122,4 +124,16 @@ describe('installDashboardCliproxyVersion', () => {
       message: 'Installed CLIProxy Plus v6.7.1, but failed to restart it',
     });
   });
+
+  it('restores a previously running proxy when installation fails', async () => {
+    const { deps, calls } = createDeps({
+      sessionRunning: true,
+      installError: new Error('checksum mismatch'),
+    });
+
+    await expect(installDashboardCliproxyVersion('6.7.1', 'plus', deps)).rejects.toThrow(
+      'checksum mismatch'
+    );
+    expect(calls.ensureCliproxyService).toBe(1);
+  });
 });