From fc56ecaac4ed8218f5d0408a6c78f173a11a78da Mon Sep 17 00:00:00 2001 From: Tam Nhu Tran Date: Mon, 10 Aug 2026 22:05:18 -0400 Subject: [PATCH] fix(cliproxy): finish concurrent update recovery --- docker/host/ccs-cliproxy-reconcile.sh | 20 ++++- docker/host/ccs-cliproxy-safe-update.sh | 45 +++++++++- src/cliproxy/binary-manager.ts | 57 ++++++++---- .../__tests__/binary-installer-atomic.test.ts | 49 +++++++++++ .../__tests__/binary-manager-install.test.ts | 4 + src/cliproxy/binary/install-lifecycle-lock.ts | 37 ++++++++ src/cliproxy/binary/installer.ts | 17 +++- src/cliproxy/binary/lifecycle.ts | 7 +- src/cliproxy/services/binary-service.ts | 49 ++++++----- .../__tests__/types-backward-compat.test.ts | 1 + src/cliproxy/types/binary-types.ts | 1 + .../cliproxy-dashboard-install-service.ts | 87 ++++++++++--------- .../cliproxy-host-continuity-assets.test.ts | 86 +++++++++++++++++- ...cliproxy-dashboard-install-service.test.ts | 55 ++++++++++++ 14 files changed, 428 insertions(+), 87 deletions(-) create mode 100644 src/cliproxy/binary/install-lifecycle-lock.ts diff --git a/docker/host/ccs-cliproxy-reconcile.sh b/docker/host/ccs-cliproxy-reconcile.sh index 7f42e062..ec67ca0b 100755 --- a/docker/host/ccs-cliproxy-reconcile.sh +++ b/docker/host/ccs-cliproxy-reconcile.sh @@ -44,6 +44,14 @@ compose_up() { -f "$compose_file" up -d --build } +compose_recreate() { + cd "$compose_dir" + docker compose --project-name "$compose_project" --project-directory "$compose_dir" \ + -f "$compose_file" up -d --force-recreate --no-build || \ + docker compose --project-name "$compose_project" --project-directory "$compose_dir" \ + -f "$compose_file" up -d --force-recreate --build +} + if ! docker inspect "$container_name" >/dev/null 2>&1; then log "Container $container_name is missing; recreating from $compose_file" compose_up @@ -79,6 +87,14 @@ if docker exec "$container_name" \ fi log 'Supervisor recovery failed; restarting the container' -docker restart "$container_name" >/dev/null +if docker restart "$container_name" >/dev/null; then + if wait_for_health; then + log "Container $container_name recovered and passed both health probes" + exit 0 + fi +fi + +log "Container $container_name is still unhealthy; recreating it from $compose_file" +compose_recreate wait_for_health -log "Container $container_name recovered and passed both health probes" +log "Container $container_name was recreated and passed both health probes" diff --git a/docker/host/ccs-cliproxy-safe-update.sh b/docker/host/ccs-cliproxy-safe-update.sh index 9dc7e963..202ea93e 100755 --- a/docker/host/ccs-cliproxy-safe-update.sh +++ b/docker/host/ccs-cliproxy-safe-update.sh @@ -55,6 +55,10 @@ stage_binary="$stage_dir/cli-proxy-api-plus" stage_version="$stage_dir/.version" backup_binary="$stage_root/previous-binary" backup_version="$stage_root/previous-version" +install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus' +install_lock_dir="$install_lock_target.lock" +install_lock_stale_seconds=600 +install_lock_owned=0 maintenance_started=0 supervisorctl_cmd() { @@ -65,6 +69,40 @@ cleanup() { rm -rf -- "$stage_root" } +remove_stale_install_lock() { + lock_mtime="$( + stat -c %Y "$install_lock_dir" 2>/dev/null || stat -f %m "$install_lock_dir" 2>/dev/null + )" || return 0 + current_time="$(date +%s)" + lock_age=$((current_time - lock_mtime)) + if [ "$lock_age" -gt "$install_lock_stale_seconds" ]; then + rmdir "$install_lock_dir" 2>/dev/null || true + fi +} + +acquire_install_lock() { + mkdir -p "$install_lock_target" + attempts=0 + while ! mkdir "$install_lock_dir" 2>/dev/null; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 240 ]; then + printf '[X] Timed out waiting for CLIProxy install lifecycle lock\n' >&2 + return 1 + fi + remove_stale_install_lock + sleep 0.25 + done + install_lock_owned=1 + touch "$install_lock_dir" +} + +release_install_lock() { + if [ "$install_lock_owned" -eq 1 ]; then + rmdir "$install_lock_dir" 2>/dev/null || true + install_lock_owned=0 + fi +} + wait_for_proxy() { attempts=0 while [ "$attempts" -lt 30 ]; do @@ -107,6 +145,7 @@ on_exit() { printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2 fi fi + release_install_lock if [ "$rollback_failed" -eq 0 ]; then cleanup else @@ -115,7 +154,10 @@ on_exit() { exit "$rc" } -trap on_exit EXIT INT TERM HUP +trap on_exit EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +trap 'exit 129' HUP mkdir -p "$stage_ccs_dir" CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest --backend plus @@ -123,6 +165,7 @@ test -x "$stage_binary" test -s "$stage_version" "$stage_binary" --version >/dev/null +acquire_install_lock cp -p "$live_binary" "$backup_binary" cp -p "$live_version" "$backup_version" diff --git a/src/cliproxy/binary-manager.ts b/src/cliproxy/binary-manager.ts index 489387fa..6698fc61 100644 --- a/src/cliproxy/binary-manager.ts +++ b/src/cliproxy/binary-manager.ts @@ -41,6 +41,7 @@ import { import type { CLIProxyBackend } from './types'; import { getVersionListCachePath } from './binary/version-cache'; import { loadOrCreateUnifiedConfig } from '../config/config-loader-facade'; +import { withInstallLifecycleLock } from './binary/install-lifecycle-lock'; export const CLIPROXY_DELETED_PLUS_REPO = 'router-for-me/CLIProxyAPIPlus'; export const CLIPROXY_PLUS_FALLBACK_TRACKING_URL = 'https://github.com/kaitranntt/ccs/issues/1062'; @@ -176,6 +177,14 @@ function getBackendBinDir(backend: CLIProxyBackend = DEFAULT_BACKEND): string { return `${baseDir}/${backend}`; } +export async function withCliproxyInstallLifecycleLock( + backend: CLIProxyBackend, + operation: () => Promise +): Promise { + const lockTarget = path.join(getBinDir(), `.install-lifecycle-${backend}`); + return withInstallLifecycleLock(lockTarget, operation); +} + /** Default configuration (uses backend from config.yaml or defaults to `DEFAULT_BACKEND`) */ function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): BinaryManagerConfig { const backendConfig = BACKEND_CONFIG[backend]; @@ -186,6 +195,7 @@ function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): Binary maxRetries: 3, verbose: false, forceVersion: false, + replaceExisting: false, skipAutoUpdate: false, allowInstall: true, backend, // Pass backend for installer to use correct download URL @@ -317,6 +327,7 @@ interface InstallCliproxyVersionDeps { formatInfo?: typeof info; formatWarn?: typeof warn; getInstalledVersion?: typeof getInstalledCliproxyVersion; + withInstallLifecycleLockFn?: typeof withCliproxyInstallLifecycleLock; } /** Install a specific version of CLIProxyAPI */ @@ -329,33 +340,41 @@ export async function installCliproxyVersion( const configuredBackend = backend ?? getConfiguredOrDefaultBackend(); const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true }); const manager = - deps.createManager?.({ version, verbose, forceVersion: true }, effectiveBackend) ?? - new BinaryManager({ version, verbose, forceVersion: true }, effectiveBackend); + deps.createManager?.( + { version, verbose, forceVersion: true, replaceExisting: true }, + effectiveBackend + ) ?? + new BinaryManager( + { version, verbose, forceVersion: true, replaceExisting: true }, + effectiveBackend + ); const stopProxyFn = deps.stopProxyFn ?? stopProxy; const waitForPortFreeFn = deps.waitForPortFreeFn ?? waitForPortFree; const formatInfo = deps.formatInfo ?? info; const formatWarn = deps.formatWarn ?? warn; + const withLifecycleLock = deps.withInstallLifecycleLockFn ?? withCliproxyInstallLifecycleLock; - // Always attempt a best-effort stop first so we also catch untracked proxies - // that are running without a session lock. - if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...')); - const result = await stopProxyFn(); - if (result.stopped) { - const stoppedPort = result.port ?? resolveLifecyclePort(); - // Wait for port to be fully released - const portFree = await waitForPortFreeFn(stoppedPort, 5000); - if (!portFree && verbose) { - console.log(formatWarn('Port did not free up in time, proceeding anyway...')); + await withLifecycleLock(effectiveBackend, async () => { + // Always attempt a best-effort stop first so we also catch untracked proxies + // that are running without a session lock. + if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...')); + const result = await stopProxyFn(); + if (result.stopped) { + const stoppedPort = result.port ?? resolveLifecyclePort(); + const portFree = await waitForPortFreeFn(stoppedPort, 5000); + if (!portFree && verbose) { + console.log(formatWarn('Port did not free up in time, proceeding anyway...')); + } + } else if (verbose && result.error && result.error !== 'No active CLIProxy session found') { + console.log(formatWarn(`Could not stop proxy: ${result.error}`)); } - } else if (verbose && result.error && result.error !== 'No active CLIProxy session found') { - console.log(formatWarn(`Could not stop proxy: ${result.error}`)); - } - await manager.ensureBinary(); + await manager.ensureBinary(); - if (verbose) { - console.log(formatInfo('New version will be active on next CLIProxy command')); - } + if (verbose) { + console.log(formatInfo('New version will be active on next CLIProxy command')); + } + }); } /** Fetch the latest CLIProxyAPI version from GitHub API */ diff --git a/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts b/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts index 5a1a95ed..3a791ad3 100644 --- a/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts +++ b/src/cliproxy/binary/__tests__/binary-installer-atomic.test.ts @@ -5,6 +5,7 @@ import * as path from 'path'; import { getExecutableName } from '../platform-detector'; import { downloadAndInstall } from '../installer'; import { ensureBinary } from '../lifecycle'; +import { withInstallLifecycleLock } from '../install-lifecycle-lock'; describe('atomic binary installation', () => { let binPath: string; @@ -78,6 +79,7 @@ describe('atomic binary installation', () => { maxRetries: 1, verbose: false, forceVersion: true, + replaceExisting: true, skipAutoUpdate: false, allowInstall: true, backend: 'original', @@ -163,6 +165,7 @@ describe('atomic binary installation', () => { maxRetries: 1, verbose: false, forceVersion: true, + replaceExisting: true, skipAutoUpdate: false, allowInstall: true, backend: 'original', @@ -178,6 +181,52 @@ describe('atomic binary installation', () => { expect(installs).toBe(1); }); + it('reuses an existing pinned binary during runtime bootstrap', async () => { + const binaryPath = path.join(binPath, getExecutableName('original')); + fs.writeFileSync(binaryPath, 'pinned-binary'); + let installs = 0; + + const resolvedPath = await ensureBinary( + { + version: '6.6.80', + releaseUrl: 'https://example.invalid', + binPath, + maxRetries: 1, + verbose: false, + forceVersion: true, + replaceExisting: false, + skipAutoUpdate: false, + allowInstall: false, + backend: 'original', + }, + { + downloadAndInstallFn: async () => { + installs += 1; + }, + } + ); + + expect(resolvedPath).toBe(binaryPath); + expect(installs).toBe(0); + }); + + it('waits for an externally held compatible install lifecycle lock', async () => { + const lockTarget = path.join(binPath, '.install-lifecycle-plus'); + fs.mkdirSync(lockTarget, { recursive: true }); + fs.mkdirSync(`${lockTarget}.lock`); + let entered = false; + + const operation = withInstallLifecycleLock(lockTarget, async () => { + entered = true; + }); + + await Bun.sleep(50); + expect(entered).toBe(false); + fs.rmdirSync(`${lockTarget}.lock`); + await operation; + expect(entered).toBe(true); + }); + it('restores the previous binary when publishing the version marker fails', async () => { const binaryName = getExecutableName('original'); const binaryPath = path.join(binPath, binaryName); diff --git a/src/cliproxy/binary/__tests__/binary-manager-install.test.ts b/src/cliproxy/binary/__tests__/binary-manager-install.test.ts index 9de84ca3..626360e3 100644 --- a/src/cliproxy/binary/__tests__/binary-manager-install.test.ts +++ b/src/cliproxy/binary/__tests__/binary-manager-install.test.ts @@ -33,6 +33,7 @@ describe('installCliproxyVersion', () => { ); await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', { + withInstallLifecycleLockFn: async (_backend, operation) => operation(), createManager: (_config: unknown, backend: string) => { seenBackend = backend; return { @@ -147,6 +148,7 @@ describe('installCliproxyVersion', () => { const binaryManager = await import(`../../binary-manager?binary-manager-install=${Date.now()}`); await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', { + withInstallLifecycleLockFn: async (_backend, operation) => operation(), createManager: () => ({ isBinaryInstalled: () => false, deleteBinary: () => { @@ -187,6 +189,7 @@ describe('installCliproxyVersion', () => { ); await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', { + withInstallLifecycleLockFn: async (_backend, operation) => operation(), createManager: () => ({ isBinaryInstalled: () => true, deleteBinary: () => { @@ -216,6 +219,7 @@ describe('installCliproxyVersion', () => { ); await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', { + withInstallLifecycleLockFn: async (_backend, operation) => operation(), createManager: () => ({ isBinaryInstalled: () => false, deleteBinary: () => undefined, diff --git a/src/cliproxy/binary/install-lifecycle-lock.ts b/src/cliproxy/binary/install-lifecycle-lock.ts new file mode 100644 index 00000000..afa534b2 --- /dev/null +++ b/src/cliproxy/binary/install-lifecycle-lock.ts @@ -0,0 +1,37 @@ +import { AsyncLocalStorage } from 'async_hooks'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as lockfile from 'proper-lockfile'; + +const heldInstallLocks = new AsyncLocalStorage>(); + +export async function withInstallLifecycleLock( + lockTarget: string, + operation: () => Promise +): Promise { + const resolvedTarget = path.resolve(lockTarget); + const held = heldInstallLocks.getStore(); + if (held?.has(resolvedTarget)) return operation(); + + fs.mkdirSync(resolvedTarget, { recursive: true }); + const release = await lockfile.lock(resolvedTarget, { + stale: 10 * 60 * 1000, + retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 }, + }); + const nextHeld = new Set(held); + nextHeld.add(resolvedTarget); + let operationError: unknown; + + try { + return await heldInstallLocks.run(nextHeld, operation); + } catch (error) { + operationError = error; + throw error; + } finally { + try { + await release(); + } catch (error) { + if (!operationError) throw error; + } + } +} diff --git a/src/cliproxy/binary/installer.ts b/src/cliproxy/binary/installer.ts index 4b1096e7..d79914c4 100644 --- a/src/cliproxy/binary/installer.ts +++ b/src/cliproxy/binary/installer.ts @@ -53,6 +53,7 @@ export async function downloadAndInstall( }); let stagingPath: string | undefined; const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`); + let installError: unknown; try { for (const entry of fs.readdirSync(config.binPath)) { @@ -130,11 +131,23 @@ export async function downloadAndInstall( spinner.succeed(`${backendLabel} ready`); console.log(ok(`${backendLabel} v${config.version} installed successfully`)); } catch (error) { + installError = error; spinner.fail('Installation failed'); throw error; } finally { - if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true }); - await releaseLock(); + let cleanupError: unknown; + try { + if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true }); + } catch (error) { + cleanupError = error; + } finally { + try { + await releaseLock(); + } catch (error) { + cleanupError ??= error; + } + } + if (!installError && cleanupError) throw cleanupError; } } diff --git a/src/cliproxy/binary/lifecycle.ts b/src/cliproxy/binary/lifecycle.ts index c5d13c94..b77eaa80 100644 --- a/src/cliproxy/binary/lifecycle.ts +++ b/src/cliproxy/binary/lifecycle.ts @@ -121,12 +121,17 @@ export async function ensureBinary( if (fs.existsSync(binaryPath)) { log(`Binary exists: ${binaryPath}`, verbose); - if (config.forceVersion) { + if (config.replaceExisting) { log(`Force version mode: installing specified version ${config.version}`, verbose); await downloadAndInstallFn(config, verbose); return binaryPath; } + if (config.forceVersion) { + log(`Pinned version mode: using installed version ${config.version}`, verbose); + return binaryPath; + } + if (config.skipAutoUpdate) { log('Runtime bootstrap mode: skipping auto-update check', verbose); return binaryPath; diff --git a/src/cliproxy/services/binary-service.ts b/src/cliproxy/services/binary-service.ts index 7c91fd09..445aab1f 100644 --- a/src/cliproxy/services/binary-service.ts +++ b/src/cliproxy/services/binary-service.ts @@ -19,6 +19,7 @@ import { clearPinnedVersion, isVersionPinned, resolveLocalBackend, + withCliproxyInstallLifecycleLock, } from '../binary-manager'; import { BACKEND_CONFIG, DEFAULT_BACKEND } from '../binary/platform-detector'; import { CLIProxyBackend } from '../types'; @@ -127,14 +128,16 @@ export async function installVersion( const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true }); try { - await installCliproxyVersion(version, verbose, effectiveBackend); - savePinnedVersion(version, effectiveBackend); + return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => { + await installCliproxyVersion(version, verbose, effectiveBackend); + savePinnedVersion(version, effectiveBackend); - return { - success: true, - version, - wasPinned: true, - }; + return { + success: true, + version, + wasPinned: true, + }; + }); } catch (error) { return { success: false, @@ -156,26 +159,28 @@ export async function installLatest( const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true }); try { - const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend); - const currentVersion = getInstalledCliproxyVersion(effectiveBackend); - const wasPinned = isVersionPinned(effectiveBackend); + return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => { + const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend); + const currentVersion = getInstalledCliproxyVersion(effectiveBackend); + const wasPinned = isVersionPinned(effectiveBackend); + + if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) { + return { + success: true, + version: latestVersion, + error: `Already running latest version: v${latestVersion}`, + }; + } + + await installCliproxyVersion(latestVersion, verbose, effectiveBackend); + clearPinnedVersion(effectiveBackend); - if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) { return { success: true, version: latestVersion, - error: `Already running latest version: v${latestVersion}`, + wasPinned, }; - } - - await installCliproxyVersion(latestVersion, verbose, effectiveBackend); - clearPinnedVersion(effectiveBackend); - - return { - success: true, - version: latestVersion, - wasPinned, - }; + }); } catch (error) { return { success: false, diff --git a/src/cliproxy/types/__tests__/types-backward-compat.test.ts b/src/cliproxy/types/__tests__/types-backward-compat.test.ts index 95952fb7..eef2705c 100644 --- a/src/cliproxy/types/__tests__/types-backward-compat.test.ts +++ b/src/cliproxy/types/__tests__/types-backward-compat.test.ts @@ -56,6 +56,7 @@ describe('types.ts backward compatibility', () => { maxRetries: 3, verbose: false, forceVersion: false, + replaceExisting: false, skipAutoUpdate: false, allowInstall: true, }; diff --git a/src/cliproxy/types/binary-types.ts b/src/cliproxy/types/binary-types.ts index 86bdbcb1..db7b1445 100644 --- a/src/cliproxy/types/binary-types.ts +++ b/src/cliproxy/types/binary-types.ts @@ -14,6 +14,7 @@ export interface BinaryManagerConfig { maxRetries: number; verbose: boolean; forceVersion: boolean; + replaceExisting?: boolean; skipAutoUpdate: boolean; allowInstall: boolean; backend?: CLIProxyBackend; diff --git a/src/web-server/services/cliproxy-dashboard-install-service.ts b/src/web-server/services/cliproxy-dashboard-install-service.ts index dd7cc9e9..1a567a0f 100644 --- a/src/web-server/services/cliproxy-dashboard-install-service.ts +++ b/src/web-server/services/cliproxy-dashboard-install-service.ts @@ -1,4 +1,8 @@ -import { installCliproxyVersion, resolveLocalBackend } from '../../cliproxy/binary-manager'; +import { + installCliproxyVersion, + resolveLocalBackend, + withCliproxyInstallLifecycleLock, +} from '../../cliproxy/binary-manager'; import { resolveLifecyclePort } from '../../cliproxy/config/port-manager'; import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/service-manager'; import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker'; @@ -25,6 +29,7 @@ interface InstallDashboardCliproxyVersionDeps { ensureCliproxyService: () => Promise; isRunningUnderSupervisord?: () => boolean; restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord; + withInstallLifecycleLock?: typeof withCliproxyInstallLifecycleLock; } const defaultDeps: InstallDashboardCliproxyVersionDeps = { @@ -75,49 +80,53 @@ export async function installDashboardCliproxyVersion( ): Promise { const effectiveBackend = resolveLocalBackend(backend, { notifyOnPlus: true }); const backendLabel = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy'; - const shouldRestoreService = await wasProxyRunning(deps); + const withLifecycleLock = deps.withInstallLifecycleLock ?? withCliproxyInstallLifecycleLock; - // The installer owns the stop-and-replace lifecycle, including best-effort - // shutdown for tracked and untracked proxies before swapping the binary. - try { - await deps.installCliproxyVersion(version, true, effectiveBackend); - } catch (error) { - if (shouldRestoreService) { - const restoreResult = await restoreProxyService(deps); - if (!restoreResult.started && !restoreResult.alreadyRunning) { - const installMessage = error instanceof Error ? error.message : String(error); - throw new ProxyError( - `${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`, - restoreResult.port - ); + return withLifecycleLock(effectiveBackend, async () => { + const shouldRestoreService = await wasProxyRunning(deps); + + // The installer owns the stop-and-replace lifecycle, including best-effort + // shutdown for tracked and untracked proxies before swapping the binary. + try { + await deps.installCliproxyVersion(version, true, effectiveBackend); + } catch (error) { + if (shouldRestoreService) { + const restoreResult = await restoreProxyService(deps); + if (!restoreResult.started && !restoreResult.alreadyRunning) { + const installMessage = error instanceof Error ? error.message : String(error); + throw new ProxyError( + `${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`, + restoreResult.port + ); + } } + throw error; + } + + if (!shouldRestoreService) { + return { + success: true, + restarted: false, + message: `Successfully installed ${backendLabel} v${version}`, + }; + } + + // In Docker, supervisord owns process lifecycle — delegate restart to it + const startResult = await restoreProxyService(deps); + if (!startResult.started && !startResult.alreadyRunning) { + return { + success: false, + restarted: false, + error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`, + message: `Installed ${backendLabel} v${version}, but failed to restart it`, + }; } - throw error; - } - if (!shouldRestoreService) { return { success: true, - restarted: false, - message: `Successfully installed ${backendLabel} v${version}`, + restarted: true, + port: startResult.port, + message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`, }; - } - - // In Docker, supervisord owns process lifecycle — delegate restart to it - const startResult = await restoreProxyService(deps); - if (!startResult.started && !startResult.alreadyRunning) { - return { - success: false, - restarted: false, - error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`, - message: `Installed ${backendLabel} v${version}, but failed to restart it`, - }; - } - - return { - success: true, - restarted: true, - port: startResult.port, - message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`, - }; + }); } diff --git a/tests/unit/docker/cliproxy-host-continuity-assets.test.ts b/tests/unit/docker/cliproxy-host-continuity-assets.test.ts index df0390e7..d2111982 100644 --- a/tests/unit/docker/cliproxy-host-continuity-assets.test.ts +++ b/tests/unit/docker/cliproxy-host-continuity-assets.test.ts @@ -1,5 +1,8 @@ -import { readFileSync } from 'fs'; +import { mkdtempSync, readFileSync, rmSync } from 'fs'; +import { spawnSync } from 'child_process'; import { describe, expect, it } from 'bun:test'; +import { tmpdir } from 'os'; +import { join } from 'path'; const updateScript = readFileSync('docker/host/ccs-cliproxy-safe-update.sh', 'utf8'); const reconcileScript = readFileSync('docker/host/ccs-cliproxy-reconcile.sh', 'utf8'); @@ -26,6 +29,54 @@ describe('CLIProxy Docker host continuity assets', () => { expect(reconcileScript).toContain('flock -n 9'); }); + it('shares the in-container install lifecycle lock with CLI and dashboard installs', () => { + expect(updateScript).toContain( + "install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'" + ); + expect(updateScript).toContain('install_lock_dir="$install_lock_target.lock"'); + expect(updateScript).toContain('while ! mkdir "$install_lock_dir"'); + expect(updateScript).toContain('touch "$install_lock_dir"'); + expect(updateScript).toContain('install_lock_stale_seconds=600'); + expect(updateScript).toContain('remove_stale_install_lock'); + expect(updateScript).toContain('rmdir "$install_lock_dir"'); + expect(updateScript.indexOf('acquire_install_lock')).toBeLessThan( + updateScript.indexOf('supervisorctl_cmd stop cliproxy') + ); + }); + + it('reclaims an orphaned stale lifecycle lock', () => { + const testRoot = mkdtempSync(join(tmpdir(), 'ccs-cliproxy-host-lock-')); + const helpersStart = updateScript.indexOf('remove_stale_install_lock() {'); + const helpersEnd = updateScript.indexOf('\nwait_for_proxy() {'); + const helpers = updateScript.slice(helpersStart, helpersEnd); + try { + const result = spawnSync( + 'bash', + [ + '-c', + `set -Eeuo pipefail +install_lock_target="$1/target" +install_lock_dir="$install_lock_target.lock" +install_lock_stale_seconds=600 +install_lock_owned=0 +mkdir -p "$install_lock_target" "$install_lock_dir" +touch -t 200001010000 "$install_lock_dir" +${helpers} +acquire_install_lock +test "$install_lock_owned" -eq 1 +release_install_lock`, + 'test-shell', + testRoot, + ], + { encoding: 'utf8' } + ); + + expect(result.status).toBe(0); + } finally { + rmSync(testRoot, { recursive: true, force: true }); + } + }); + it('rolls back failed swaps and health-checks both services', () => { expect(updateScript).toContain('rollback()'); expect(updateScript).toContain('previous-binary'); @@ -41,6 +92,7 @@ describe('CLIProxy Docker host continuity assets', () => { expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build'); expect(reconcileScript).toContain('restart ccs-dashboard cliproxy'); expect(reconcileScript).toContain('docker restart "$container_name"'); + expect(reconcileScript).toContain('up -d --force-recreate --no-build'); expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan( reconcileScript.indexOf('restart ccs-dashboard cliproxy') ); @@ -53,4 +105,36 @@ describe('CLIProxy Docker host continuity assets', () => { expect(reconcileService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh'); expect(reconcileTimer).toContain('OnUnitActiveSec=30s'); }); + + it('forces nonzero signal exits so the EXIT trap rolls back maintenance', () => { + expect(updateScript).toContain("trap 'exit 130' INT"); + expect(updateScript).toContain("trap 'exit 143' TERM"); + expect(updateScript).toContain("trap 'exit 129' HUP"); + + const handlerStart = updateScript.indexOf('on_exit() {'); + const handlerEnd = updateScript.indexOf('\ntrap on_exit EXIT'); + const handlers = updateScript.slice(handlerStart, handlerEnd); + const result = spawnSync( + 'bash', + [ + '-c', + `set -Eeuo pipefail +maintenance_started=1 +stage_root=/tmp/unused +rollback() { printf 'rollback\\n'; } +cleanup() { printf 'cleanup\\n'; } +release_install_lock() { :; } +${handlers} +trap on_exit EXIT +trap 'exit 143' TERM +true +kill -TERM $$`, + ], + { encoding: 'utf8' } + ); + + expect(result.status).toBe(143); + expect(result.stdout).toContain('rollback'); + expect(result.stdout).toContain('cleanup'); + }); }); diff --git a/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts b/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts index fc6a61cf..c634d6f1 100644 --- a/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts +++ b/tests/unit/web-server/cliproxy-dashboard-install-service.test.ts @@ -43,6 +43,10 @@ function createDeps( } ); }, + withInstallLifecycleLock: async ( + _backend: CLIProxyBackend, + operation: () => Promise + ) => operation(), }; return { deps, calls }; @@ -136,4 +140,55 @@ describe('installDashboardCliproxyVersion', () => { ); expect(calls.ensureCliproxyService).toBe(1); }); + + it('serializes concurrent dashboard stop-install-restore transactions', async () => { + let running = true; + let queue = Promise.resolve(); + let activeTransactions = 0; + let maxActiveTransactions = 0; + let restores = 0; + + const deps = { + getProxyStatus: () => ({ running }), + isCliproxyRunning: async () => running, + installCliproxyVersion: async () => { + running = false; + await new Promise((resolve) => setTimeout(resolve, 5)); + }, + ensureCliproxyService: async () => { + running = true; + restores += 1; + return { started: true, alreadyRunning: false, port: 8317 }; + }, + withInstallLifecycleLock: ( + _backend: CLIProxyBackend, + operation: () => Promise + ): Promise => { + const result = queue.then(async () => { + activeTransactions += 1; + maxActiveTransactions = Math.max(maxActiveTransactions, activeTransactions); + try { + return await operation(); + } finally { + activeTransactions -= 1; + } + }); + queue = result.then( + () => undefined, + () => undefined + ); + return result; + }, + }; + + const results = await Promise.all([ + installDashboardCliproxyVersion('6.7.1', 'plus', deps), + installDashboardCliproxyVersion('6.7.2', 'plus', deps), + ]); + + expect(maxActiveTransactions).toBe(1); + expect(restores).toBe(2); + expect(results.every((result) => result.restarted)).toBe(true); + expect(running).toBe(true); + }); });