diff --git a/docker/host/ccs-cliproxy-reconcile.sh b/docker/host/ccs-cliproxy-reconcile.sh index 0363f449..7f42e062 100755 --- a/docker/host/ccs-cliproxy-reconcile.sh +++ b/docker/host/ccs-cliproxy-reconcile.sh @@ -3,8 +3,9 @@ set -Eeuo pipefail container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}" -compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker}" -compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.integrated.yml}" +compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/opt/cliproxy}" +compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.yml}" +compose_project="${CCS_CLIPROXY_COMPOSE_PROJECT:-docker}" lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}" log_file="${CCS_CLIPROXY_RECONCILE_LOG:-/var/log/ccs-cliproxy-reconcile.log}" @@ -37,8 +38,10 @@ wait_for_health() { compose_up() { cd "$compose_dir" - docker compose -f "$compose_file" up -d --no-build || \ - docker compose -f "$compose_file" up -d --build + docker compose --project-name "$compose_project" --project-directory "$compose_dir" \ + -f "$compose_file" up -d --no-build || \ + docker compose --project-name "$compose_project" --project-directory "$compose_dir" \ + -f "$compose_file" up -d --build } if ! docker inspect "$container_name" >/dev/null 2>&1; then @@ -67,10 +70,12 @@ if probe; then fi log 'Dashboard or proxy failed two consecutive probes; restarting supervised processes' -docker exec "$container_name" supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy -if wait_for_health; then - log 'Supervised processes recovered and passed both health probes' - exit 0 +if docker exec "$container_name" \ + supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy; then + if wait_for_health; then + log 'Supervised processes recovered and passed both health probes' + exit 0 + fi fi log 'Supervisor recovery failed; restarting the container' diff --git a/docker/host/ccs-cliproxy-safe-update.sh b/docker/host/ccs-cliproxy-safe-update.sh index 63f11065..9dc7e963 100755 --- a/docker/host/ccs-cliproxy-safe-update.sh +++ b/docker/host/ccs-cliproxy-safe-update.sh @@ -26,11 +26,16 @@ if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'tru exit 1 fi -status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)" || { +status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus --verbose 2>&1)" || { log "Unable to inspect the installed CLIProxy version: $status_output" exit 1 } +if grep -qi 'Could not fetch' <<<"$status_output"; then + log "Unable to check the latest CLIProxy version: $(grep -im1 'Could not fetch' <<<"$status_output" | xargs)" + exit 1 +fi + if ! grep -qi 'update available' <<<"$status_output"; then exit 0 fi @@ -50,7 +55,7 @@ stage_binary="$stage_dir/cli-proxy-api-plus" stage_version="$stage_dir/.version" backup_binary="$stage_root/previous-binary" backup_version="$stage_root/previous-version" -swap_started=0 +maintenance_started=0 supervisorctl_cmd() { supervisorctl -c /etc/supervisord.conf "$@" @@ -73,26 +78,40 @@ wait_for_proxy() { } rollback() { + rollback_ok=1 supervisorctl_cmd stop cliproxy >/dev/null 2>&1 || true if [ -f "$backup_binary" ]; then - install -m 0755 "$backup_binary" "$live_binary.rollback" - mv -f "$live_binary.rollback" "$live_binary" + install -m 0755 "$backup_binary" "$live_binary.rollback" && \ + mv -f "$live_binary.rollback" "$live_binary" || rollback_ok=0 + else + rollback_ok=0 fi if [ -f "$backup_version" ]; then - install -m 0644 "$backup_version" "$live_version.rollback" - mv -f "$live_version.rollback" "$live_version" + install -m 0644 "$backup_version" "$live_version.rollback" && \ + mv -f "$live_version.rollback" "$live_version" || rollback_ok=0 + else + rollback_ok=0 fi - supervisorctl_cmd start cliproxy >/dev/null - wait_for_proxy + supervisorctl_cmd start cliproxy >/dev/null || rollback_ok=0 + wait_for_proxy || rollback_ok=0 + [ "$rollback_ok" -eq 1 ] } on_exit() { rc=$? trap - EXIT INT TERM HUP - if [ "$rc" -ne 0 ] && [ "$swap_started" -eq 1 ]; then - rollback || true + rollback_failed=0 + if [ "$rc" -ne 0 ] && [ "$maintenance_started" -eq 1 ]; then + if ! rollback; then + rollback_failed=1 + printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2 + fi + fi + if [ "$rollback_failed" -eq 0 ]; then + cleanup + else + rc=70 fi - cleanup exit "$rc" } @@ -107,17 +126,17 @@ test -s "$stage_version" cp -p "$live_binary" "$backup_binary" cp -p "$live_version" "$backup_version" +maintenance_started=1 supervisorctl_cmd stop cliproxy >/dev/null -swap_started=1 mv -f "$stage_binary" "$live_binary" mv -f "$stage_version" "$live_version" chmod 0755 "$live_binary" supervisorctl_cmd start cliproxy >/dev/null wait_for_proxy -swap_started=0 +maintenance_started=0 CONTAINER_UPDATE then - log 'CLIProxy Plus update failed; previous binary was restored and restarted' + log 'CLIProxy Plus update failed; rollback was attempted and reconciliation will verify service health' exit 1 fi diff --git a/docker/host/systemd/ccs-cliproxy-reconcile.service b/docker/host/systemd/ccs-cliproxy-reconcile.service index 0ff3730c..b1ec3961 100644 --- a/docker/host/systemd/ccs-cliproxy-reconcile.service +++ b/docker/host/systemd/ccs-cliproxy-reconcile.service @@ -6,4 +6,4 @@ Requires=docker.service [Service] Type=oneshot ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh -TimeoutStartSec=5min +TimeoutStartSec=10min diff --git a/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts b/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts index 3cc16c26..5a1a95ed 100644 --- a/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts +++ b/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts @@ -4,6 +4,7 @@ import * as os from 'os'; import * as path from 'path'; import { getExecutableName } from '../platform-detector'; import { downloadAndInstall } from '../installer'; +import { ensureBinary } from '../lifecycle'; describe('atomic binary installation', () => { let binPath: string; @@ -148,4 +149,162 @@ describe('atomic binary installation', () => { expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80'); expect(stagingDirectories()).toEqual([]); }); + + it('installs a forced version even when an older binary already exists', async () => { + const binaryPath = path.join(binPath, getExecutableName('original')); + fs.writeFileSync(binaryPath, 'old-binary'); + let installs = 0; + + const resolvedPath = await ensureBinary( + { + version: '6.7.1', + releaseUrl: 'https://example.invalid', + binPath, + maxRetries: 1, + verbose: false, + forceVersion: true, + skipAutoUpdate: false, + allowInstall: true, + backend: 'original', + }, + { + downloadAndInstallFn: async () => { + installs += 1; + }, + } + ); + + expect(resolvedPath).toBe(binaryPath); + expect(installs).toBe(1); + }); + + it('restores the previous binary when publishing the version marker fails', async () => { + const binaryName = getExecutableName('original'); + const binaryPath = path.join(binPath, binaryName); + const versionPath = path.join(binPath, '.version'); + fs.writeFileSync(binaryPath, 'old-binary'); + fs.writeFileSync(versionPath, '6.6.80'); + let renames = 0; + + await expect( + downloadAndInstall( + { + version: '6.7.1', + releaseUrl: 'https://example.invalid', + binPath, + maxRetries: 1, + verbose: false, + forceVersion: true, + skipAutoUpdate: false, + allowInstall: true, + backend: 'original', + }, + false, + { + downloadWithRetryFn: async (_url, archivePath) => { + fs.writeFileSync(archivePath, 'downloaded-archive'); + return { success: true, filePath: archivePath, retries: 0 }; + }, + verifyChecksumFn: async () => ({ + valid: true, + expected: 'checksum', + actual: 'checksum', + }), + extractArchiveFn: async (_archivePath, destination) => { + fs.writeFileSync(path.join(destination, binaryName), 'new-binary'); + }, + renameSyncFn: (source, destination) => { + renames += 1; + if (renames === 2) throw new Error('version marker blocked'); + fs.renameSync(source, destination); + }, + } + ) + ).rejects.toThrow('version marker blocked'); + + expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary'); + expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80'); + expect(stagingDirectories()).toEqual([]); + }); + + it('serializes concurrent installs so the binary and version stay paired', async () => { + const binaryName = getExecutableName('original'); + const binaryPath = path.join(binPath, binaryName); + const versionPath = path.join(binPath, '.version'); + + const install = (version: string) => + downloadAndInstall( + { + version, + releaseUrl: 'https://example.invalid', + binPath, + maxRetries: 1, + verbose: false, + forceVersion: true, + skipAutoUpdate: false, + allowInstall: true, + backend: 'original', + }, + false, + { + downloadWithRetryFn: async (_url, archivePath) => { + fs.writeFileSync(archivePath, 'downloaded-archive'); + return { success: true, filePath: archivePath, retries: 0 }; + }, + verifyChecksumFn: async () => ({ + valid: true, + expected: 'checksum', + actual: 'checksum', + }), + extractArchiveFn: async (_archivePath, destination) => { + fs.writeFileSync(path.join(destination, binaryName), `binary-${version}`); + }, + } + ); + + await Promise.all([install('6.7.1'), install('6.7.2')]); + + const installedVersion = fs.readFileSync(versionPath, 'utf8'); + expect(fs.readFileSync(binaryPath, 'utf8')).toBe(`binary-${installedVersion}`); + expect(stagingDirectories()).toEqual([]); + }); + + it('removes staging residue left by an interrupted prior install', async () => { + const stalePath = path.join(binPath, '.cliproxy-install-stale'); + fs.mkdirSync(stalePath); + fs.writeFileSync(path.join(stalePath, 'partial-archive'), 'partial'); + const binaryName = getExecutableName('original'); + + await downloadAndInstall( + { + version: '6.7.1', + releaseUrl: 'https://example.invalid', + binPath, + maxRetries: 1, + verbose: false, + forceVersion: true, + skipAutoUpdate: false, + allowInstall: true, + backend: 'original', + }, + false, + { + downloadWithRetryFn: async (_url, archivePath) => { + fs.writeFileSync(archivePath, 'downloaded-archive'); + return { success: true, filePath: archivePath, retries: 0 }; + }, + verifyChecksumFn: async () => ({ + valid: true, + expected: 'checksum', + actual: 'checksum', + }), + extractArchiveFn: async (_archivePath, destination) => { + fs.writeFileSync(path.join(destination, binaryName), 'new-binary'); + }, + } + ); + + expect(fs.existsSync(stalePath)).toBe(false); + expect(stagingDirectories()).toEqual([]); + }); }); diff --git a/src/cliproxy/binary/installer.ts b/src/cliproxy/binary/installer.ts index db22362c..4b1096e7 100644 --- a/src/cliproxy/binary/installer.ts +++ b/src/cliproxy/binary/installer.ts @@ -5,6 +5,7 @@ import * as fs from 'fs'; import * as path from 'path'; +import * as lockfile from 'proper-lockfile'; import { BinaryManagerConfig } from '../types'; import { detectPlatform, @@ -16,7 +17,6 @@ import { import { downloadWithRetry } from './downloader'; import { verifyChecksum, computeChecksum } from './verifier'; import { extractArchive } from './extractor'; -import { writeInstalledVersion } from './version-cache'; import { ProgressIndicator } from '../../utils/progress-indicator'; import { ok } from '../../utils/ui'; import { BinaryError, NetworkError } from '../../errors/error-types'; @@ -47,14 +47,29 @@ export async function downloadAndInstall( const renameSyncFn = deps.renameSyncFn ?? fs.renameSync; fs.mkdirSync(config.binPath, { recursive: true }); - const stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-')); - const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`); - const stagedBinary = path.join(stagingPath, getExecutableName(backend)); - const installedBinary = path.join(config.binPath, getExecutableName(backend)); + const releaseLock = await lockfile.lock(config.binPath, { + stale: 10 * 60 * 1000, + retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 }, + }); + let stagingPath: string | undefined; const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`); - spinner.start(); try { + for (const entry of fs.readdirSync(config.binPath)) { + if (entry.startsWith('.cliproxy-install-')) { + fs.rmSync(path.join(config.binPath, entry), { recursive: true, force: true }); + } + } + stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-')); + const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`); + const stagedBinary = path.join(stagingPath, getExecutableName(backend)); + const stagedVersion = path.join(stagingPath, '.version'); + const installedBinary = path.join(config.binPath, getExecutableName(backend)); + const installedVersion = path.join(config.binPath, '.version'); + const backupBinary = path.join(stagingPath, '.previous-binary'); + const hadInstalledBinary = fs.existsSync(installedBinary); + spinner.start(); + const result = await downloadWithRetryFn(downloadUrl, archivePath, { maxRetries: config.maxRetries, verbose, @@ -95,15 +110,31 @@ export async function downloadAndInstall( if (verbose) console.error(`[cliproxy] Set executable permissions: ${stagedBinary}`); } + fs.writeFileSync(stagedVersion, config.version, 'utf8'); + if (hadInstalledBinary) { + fs.copyFileSync(installedBinary, backupBinary); + if (platform.os !== 'windows') fs.chmodSync(backupBinary, 0o755); + } + renameSyncFn(stagedBinary, installedBinary); - writeInstalledVersion(config.binPath, config.version); + try { + renameSyncFn(stagedVersion, installedVersion); + } catch (error) { + if (hadInstalledBinary) { + renameSyncFn(backupBinary, installedBinary); + } else { + fs.unlinkSync(installedBinary); + } + throw error; + } spinner.succeed(`${backendLabel} ready`); console.log(ok(`${backendLabel} v${config.version} installed successfully`)); } catch (error) { spinner.fail('Installation failed'); throw error; } finally { - fs.rmSync(stagingPath, { recursive: true, force: true }); + if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true }); + await releaseLock(); } } diff --git a/src/cliproxy/binary/lifecycle.ts b/src/cliproxy/binary/lifecycle.ts index e9ea3802..c5d13c94 100644 --- a/src/cliproxy/binary/lifecycle.ts +++ b/src/cliproxy/binary/lifecycle.ts @@ -15,6 +15,7 @@ import { downloadAndInstall, getBinaryPath } from './installer'; import { info, warn } from '../../utils/ui'; import { isCliproxyRunning } from '../services/stats-fetcher'; import { resolveLifecyclePort } from '../config/port-manager'; +import { BinaryError } from '../../errors/error-types'; import { CLIPROXY_MAX_STABLE_VERSION, CLIPROXY_FAULTY_RANGE, @@ -103,17 +104,26 @@ async function handleAutoUpdate(config: BinaryManagerConfig, verbose: boolean): * Ensure binary is available (download if missing, update if outdated) * @returns Path to executable binary */ -export async function ensureBinary(config: BinaryManagerConfig): Promise { +interface EnsureBinaryDeps { + downloadAndInstallFn?: typeof downloadAndInstall; +} + +export async function ensureBinary( + config: BinaryManagerConfig, + deps: EnsureBinaryDeps = {} +): Promise { const verbose = config.verbose; const backend: CLIProxyBackend = config.backend ?? DEFAULT_BACKEND; const binaryPath = getBinaryPath(config.binPath, backend); + const downloadAndInstallFn = deps.downloadAndInstallFn ?? downloadAndInstall; // Binary exists - check for updates unless forceVersion if (fs.existsSync(binaryPath)) { log(`Binary exists: ${binaryPath}`, verbose); if (config.forceVersion) { - log('Force version mode: skipping auto-update', verbose); + log(`Force version mode: installing specified version ${config.version}`, verbose); + await downloadAndInstallFn(config, verbose); return binaryPath; } @@ -134,9 +144,10 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise // Binary missing if (!config.allowInstall) { - throw new Error( + throw new BinaryError( `${getBackendLabel(backend)} binary is not installed locally. ` + - 'Run "ccs cliproxy install" when you have network access.' + 'Run "ccs cliproxy install" when you have network access.', + binaryPath ); } @@ -161,6 +172,6 @@ export async function ensureBinary(config: BinaryManagerConfig): Promise log(`Force version mode: using specified version ${config.version}`, verbose); } - await downloadAndInstall(config, verbose); + await downloadAndInstallFn(config, verbose); return binaryPath; } diff --git a/src/web-server/services/cliproxy-dashboard-install-service.ts b/src/web-server/services/cliproxy-dashboard-install-service.ts index 029bd472..dd7cc9e9 100644 --- a/src/web-server/services/cliproxy-dashboard-install-service.ts +++ b/src/web-server/services/cliproxy-dashboard-install-service.ts @@ -4,6 +4,7 @@ import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/s import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker'; import { isCliproxyRunning } from '../../cliproxy/services/stats-fetcher'; import type { CLIProxyBackend } from '../../cliproxy/types'; +import { ProxyError } from '../../errors/error-types'; import { isRunningUnderSupervisord, restartCliproxyViaSupervisord, @@ -22,6 +23,8 @@ interface InstallDashboardCliproxyVersionDeps { backend?: CLIProxyBackend ) => Promise; ensureCliproxyService: () => Promise; + isRunningUnderSupervisord?: () => boolean; + restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord; } const defaultDeps: InstallDashboardCliproxyVersionDeps = { @@ -48,6 +51,23 @@ async function wasProxyRunning(deps: InstallDashboardCliproxyVersionDeps): Promi return deps.isCliproxyRunning(); } +async function restoreProxyService( + deps: InstallDashboardCliproxyVersionDeps +): Promise { + const underSupervisord = deps.isRunningUnderSupervisord?.() ?? isRunningUnderSupervisord(); + if (underSupervisord) { + const restart = deps.restartCliproxyViaSupervisord?.() ?? restartCliproxyViaSupervisord(); + return { + started: restart.success, + alreadyRunning: false, + port: restart.port ?? resolveLifecyclePort(), + error: restart.error, + }; + } + + return deps.ensureCliproxyService(); +} + export async function installDashboardCliproxyVersion( version: string, backend: CLIProxyBackend, @@ -59,7 +79,21 @@ export async function installDashboardCliproxyVersion( // The installer owns the stop-and-replace lifecycle, including best-effort // shutdown for tracked and untracked proxies before swapping the binary. - await deps.installCliproxyVersion(version, true, effectiveBackend); + try { + await deps.installCliproxyVersion(version, true, effectiveBackend); + } catch (error) { + if (shouldRestoreService) { + const restoreResult = await restoreProxyService(deps); + if (!restoreResult.started && !restoreResult.alreadyRunning) { + const installMessage = error instanceof Error ? error.message : String(error); + throw new ProxyError( + `${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`, + restoreResult.port + ); + } + } + throw error; + } if (!shouldRestoreService) { return { @@ -70,20 +104,7 @@ export async function installDashboardCliproxyVersion( } // In Docker, supervisord owns process lifecycle — delegate restart to it - if (isRunningUnderSupervisord()) { - const result = restartCliproxyViaSupervisord(); - return { - success: result.success, - restarted: result.success, - port: result.port, - error: result.error, - message: result.success - ? `Successfully installed ${backendLabel} v${version} and restarted it on port ${result.port}` - : `Installed ${backendLabel} v${version}, but restart failed`, - }; - } - - const startResult = await deps.ensureCliproxyService(); + const startResult = await restoreProxyService(deps); if (!startResult.started && !startResult.alreadyRunning) { return { success: false, diff --git a/tests/unit/docker/cliproxy-host-continuity-assets.test.ts b/tests/unit/docker/cliproxy-host-continuity-assets.test.ts index dee2a1e3..df0390e7 100644 --- a/tests/unit/docker/cliproxy-host-continuity-assets.test.ts +++ b/tests/unit/docker/cliproxy-host-continuity-assets.test.ts @@ -35,9 +35,15 @@ describe('CLIProxy Docker host continuity assets', () => { }); it('recreates missing containers and escalates unhealthy recovery', () => { - expect(reconcileScript).toContain('docker compose -f "$compose_file" up -d --no-build'); + expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_DIR:-/opt/cliproxy'); + expect(reconcileScript).toContain('CCS_CLIPROXY_COMPOSE_PROJECT:-docker'); + expect(reconcileScript).toContain('--project-name "$compose_project"'); + expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build'); expect(reconcileScript).toContain('restart ccs-dashboard cliproxy'); expect(reconcileScript).toContain('docker restart "$container_name"'); + expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan( + reconcileScript.indexOf('restart ccs-dashboard cliproxy') + ); }); it('installs executable services on bounded timers', () => { diff --git a/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts b/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts index 5805ed54..fc6a61cf 100644 --- a/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts +++ b/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts @@ -10,6 +10,7 @@ function createDeps( sessionRunning?: boolean; remoteRunning?: boolean; startResult?: { started: boolean; alreadyRunning: boolean; port: number; error?: string }; + installError?: Error; } = {} ) { const calls = { @@ -30,6 +31,7 @@ function createDeps( _backend?: CLIProxyBackend ) => { calls.installCliproxyVersion += 1; + if (overrides.installError) throw overrides.installError; }, ensureCliproxyService: async () => { calls.ensureCliproxyService += 1; @@ -122,4 +124,16 @@ describe('installDashboardCliproxyVersion', () => { message: 'Installed CLIProxy Plus v6.7.1, but failed to restart it', }); }); + + it('restores a previously running proxy when installation fails', async () => { + const { deps, calls } = createDeps({ + sessionRunning: true, + installError: new Error('checksum mismatch'), + }); + + await expect(installDashboardCliproxyVersion('6.7.1', 'plus', deps)).rejects.toThrow( + 'checksum mismatch' + ); + expect(calls.ensureCliproxyService).toBe(1); + }); });