fix(cliproxy): finish concurrent update recovery

This commit is contained in:
Tam Nhu Tran committed 2026-08-10 22:13:29 -04:00
1 parent 192b27fbb2
commit fc56ecaac4
14 files changed
+428 -87

No files matched your search

+18 -2
View File
@@ -44,6 +44,14 @@ compose_up() {
-f "$compose_file" up -d --build
}
compose_recreate() {
cd "$compose_dir"
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --force-recreate --no-build || \
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
-f "$compose_file" up -d --force-recreate --build
}
if ! docker inspect "$container_name" >/dev/null 2>&1; then
log "Container $container_name is missing; recreating from $compose_file"
compose_up
@@ -79,6 +87,14 @@ if docker exec "$container_name" \
fi
log 'Supervisor recovery failed; restarting the container'
docker restart "$container_name" >/dev/null
if docker restart "$container_name" >/dev/null; then
if wait_for_health; then
log "Container $container_name recovered and passed both health probes"
exit 0
fi
fi
log "Container $container_name is still unhealthy; recreating it from $compose_file"
compose_recreate
wait_for_health
log "Container $container_name recovered and passed both health probes"
log "Container $container_name was recreated and passed both health probes"
+44 -1
View File
@@ -55,6 +55,10 @@ stage_binary="$stage_dir/cli-proxy-api-plus"
stage_version="$stage_dir/.version"
backup_binary="$stage_root/previous-binary"
backup_version="$stage_root/previous-version"
install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'
install_lock_dir="$install_lock_target.lock"
install_lock_stale_seconds=600
install_lock_owned=0
maintenance_started=0
supervisorctl_cmd() {
@@ -65,6 +69,40 @@ cleanup() {
rm -rf -- "$stage_root"
}
remove_stale_install_lock() {
lock_mtime="$(
stat -c %Y "$install_lock_dir" 2>/dev/null || stat -f %m "$install_lock_dir" 2>/dev/null
)" || return 0
current_time="$(date +%s)"
lock_age=$((current_time - lock_mtime))
if [ "$lock_age" -gt "$install_lock_stale_seconds" ]; then
rmdir "$install_lock_dir" 2>/dev/null || true
fi
}
acquire_install_lock() {
mkdir -p "$install_lock_target"
attempts=0
while ! mkdir "$install_lock_dir" 2>/dev/null; do
attempts=$((attempts + 1))
if [ "$attempts" -ge 240 ]; then
printf '[X] Timed out waiting for CLIProxy install lifecycle lock\n' >&2
return 1
fi
remove_stale_install_lock
sleep 0.25
done
install_lock_owned=1
touch "$install_lock_dir"
}
release_install_lock() {
if [ "$install_lock_owned" -eq 1 ]; then
rmdir "$install_lock_dir" 2>/dev/null || true
install_lock_owned=0
fi
}
wait_for_proxy() {
attempts=0
while [ "$attempts" -lt 30 ]; do
@@ -107,6 +145,7 @@ on_exit() {
printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2
fi
fi
release_install_lock
if [ "$rollback_failed" -eq 0 ]; then
cleanup
else
@@ -115,7 +154,10 @@ on_exit() {
exit "$rc"
}
trap on_exit EXIT INT TERM HUP
trap on_exit EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
trap 'exit 129' HUP
mkdir -p "$stage_ccs_dir"
CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest --backend plus
@@ -123,6 +165,7 @@ test -x "$stage_binary"
test -s "$stage_version"
"$stage_binary" --version >/dev/null
acquire_install_lock
cp -p "$live_binary" "$backup_binary"
cp -p "$live_version" "$backup_version"
+38 -19
View File
@@ -41,6 +41,7 @@ import {
import type { CLIProxyBackend } from './types';
import { getVersionListCachePath } from './binary/version-cache';
import { loadOrCreateUnifiedConfig } from '../config/config-loader-facade';
import { withInstallLifecycleLock } from './binary/install-lifecycle-lock';
export const CLIPROXY_DELETED_PLUS_REPO = 'router-for-me/CLIProxyAPIPlus';
export const CLIPROXY_PLUS_FALLBACK_TRACKING_URL = 'https://github.com/kaitranntt/ccs/issues/1062';
@@ -176,6 +177,14 @@ function getBackendBinDir(backend: CLIProxyBackend = DEFAULT_BACKEND): string {
return `${baseDir}/${backend}`;
}
export async function withCliproxyInstallLifecycleLock<T>(
backend: CLIProxyBackend,
operation: () => Promise<T>
): Promise<T> {
const lockTarget = path.join(getBinDir(), `.install-lifecycle-${backend}`);
return withInstallLifecycleLock(lockTarget, operation);
}
/** Default configuration (uses backend from config.yaml or defaults to `DEFAULT_BACKEND`) */
function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): BinaryManagerConfig {
const backendConfig = BACKEND_CONFIG[backend];
@@ -186,6 +195,7 @@ function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): Binary
maxRetries: 3,
verbose: false,
forceVersion: false,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: true,
backend, // Pass backend for installer to use correct download URL
@@ -317,6 +327,7 @@ interface InstallCliproxyVersionDeps {
formatInfo?: typeof info;
formatWarn?: typeof warn;
getInstalledVersion?: typeof getInstalledCliproxyVersion;
withInstallLifecycleLockFn?: typeof withCliproxyInstallLifecycleLock;
}
/** Install a specific version of CLIProxyAPI */
@@ -329,33 +340,41 @@ export async function installCliproxyVersion(
const configuredBackend = backend ?? getConfiguredOrDefaultBackend();
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
const manager =
deps.createManager?.({ version, verbose, forceVersion: true }, effectiveBackend) ??
new BinaryManager({ version, verbose, forceVersion: true }, effectiveBackend);
deps.createManager?.(
{ version, verbose, forceVersion: true, replaceExisting: true },
effectiveBackend
) ??
new BinaryManager(
{ version, verbose, forceVersion: true, replaceExisting: true },
effectiveBackend
);
const stopProxyFn = deps.stopProxyFn ?? stopProxy;
const waitForPortFreeFn = deps.waitForPortFreeFn ?? waitForPortFree;
const formatInfo = deps.formatInfo ?? info;
const formatWarn = deps.formatWarn ?? warn;
const withLifecycleLock = deps.withInstallLifecycleLockFn ?? withCliproxyInstallLifecycleLock;
// Always attempt a best-effort stop first so we also catch untracked proxies
// that are running without a session lock.
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
const result = await stopProxyFn();
if (result.stopped) {
const stoppedPort = result.port ?? resolveLifecyclePort();
// Wait for port to be fully released
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
if (!portFree && verbose) {
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
await withLifecycleLock(effectiveBackend, async () => {
// Always attempt a best-effort stop first so we also catch untracked proxies
// that are running without a session lock.
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
const result = await stopProxyFn();
if (result.stopped) {
const stoppedPort = result.port ?? resolveLifecyclePort();
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
if (!portFree && verbose) {
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
}
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
}
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
}
await manager.ensureBinary();
await manager.ensureBinary();
if (verbose) {
console.log(formatInfo('New version will be active on next CLIProxy command'));
}
if (verbose) {
console.log(formatInfo('New version will be active on next CLIProxy command'));
}
});
}
/** Fetch the latest CLIProxyAPI version from GitHub API */
@@ -5,6 +5,7 @@ import * as path from 'path';
import { getExecutableName } from '../platform-detector';
import { downloadAndInstall } from '../installer';
import { ensureBinary } from '../lifecycle';
import { withInstallLifecycleLock } from '../install-lifecycle-lock';
describe('atomic binary installation', () => {
let binPath: string;
@@ -78,6 +79,7 @@ describe('atomic binary installation', () => {
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
@@ -163,6 +165,7 @@ describe('atomic binary installation', () => {
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
@@ -178,6 +181,52 @@ describe('atomic binary installation', () => {
expect(installs).toBe(1);
});
it('reuses an existing pinned binary during runtime bootstrap', async () => {
const binaryPath = path.join(binPath, getExecutableName('original'));
fs.writeFileSync(binaryPath, 'pinned-binary');
let installs = 0;
const resolvedPath = await ensureBinary(
{
version: '6.6.80',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: false,
backend: 'original',
},
{
downloadAndInstallFn: async () => {
installs += 1;
},
}
);
expect(resolvedPath).toBe(binaryPath);
expect(installs).toBe(0);
});
it('waits for an externally held compatible install lifecycle lock', async () => {
const lockTarget = path.join(binPath, '.install-lifecycle-plus');
fs.mkdirSync(lockTarget, { recursive: true });
fs.mkdirSync(`${lockTarget}.lock`);
let entered = false;
const operation = withInstallLifecycleLock(lockTarget, async () => {
entered = true;
});
await Bun.sleep(50);
expect(entered).toBe(false);
fs.rmdirSync(`${lockTarget}.lock`);
await operation;
expect(entered).toBe(true);
});
it('restores the previous binary when publishing the version marker fails', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
@@ -33,6 +33,7 @@ describe('installCliproxyVersion', () => {
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: (_config: unknown, backend: string) => {
seenBackend = backend;
return {
@@ -147,6 +148,7 @@ describe('installCliproxyVersion', () => {
const binaryManager = await import(`../../binary-manager?binary-manager-install=${Date.now()}`);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => false,
deleteBinary: () => {
@@ -187,6 +189,7 @@ describe('installCliproxyVersion', () => {
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => true,
deleteBinary: () => {
@@ -216,6 +219,7 @@ describe('installCliproxyVersion', () => {
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
createManager: () => ({
isBinaryInstalled: () => false,
deleteBinary: () => undefined,
@@ -0,0 +1,37 @@
import { AsyncLocalStorage } from 'async_hooks';
import * as fs from 'fs';
import * as path from 'path';
import * as lockfile from 'proper-lockfile';
const heldInstallLocks = new AsyncLocalStorage<Set<string>>();
export async function withInstallLifecycleLock<T>(
lockTarget: string,
operation: () => Promise<T>
): Promise<T> {
const resolvedTarget = path.resolve(lockTarget);
const held = heldInstallLocks.getStore();
if (held?.has(resolvedTarget)) return operation();
fs.mkdirSync(resolvedTarget, { recursive: true });
const release = await lockfile.lock(resolvedTarget, {
stale: 10 * 60 * 1000,
retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 },
});
const nextHeld = new Set(held);
nextHeld.add(resolvedTarget);
let operationError: unknown;
try {
return await heldInstallLocks.run(nextHeld, operation);
} catch (error) {
operationError = error;
throw error;
} finally {
try {
await release();
} catch (error) {
if (!operationError) throw error;
}
}
}
+15 -2
View File
@@ -53,6 +53,7 @@ export async function downloadAndInstall(
});
let stagingPath: string | undefined;
const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`);
let installError: unknown;
try {
for (const entry of fs.readdirSync(config.binPath)) {
@@ -130,11 +131,23 @@ export async function downloadAndInstall(
spinner.succeed(`${backendLabel} ready`);
console.log(ok(`${backendLabel} v${config.version} installed successfully`));
} catch (error) {
installError = error;
spinner.fail('Installation failed');
throw error;
} finally {
if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
await releaseLock();
let cleanupError: unknown;
try {
if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
} catch (error) {
cleanupError = error;
} finally {
try {
await releaseLock();
} catch (error) {
cleanupError ??= error;
}
}
if (!installError && cleanupError) throw cleanupError;
}
}
+6 -1
View File
@@ -121,12 +121,17 @@ export async function ensureBinary(
if (fs.existsSync(binaryPath)) {
log(`Binary exists: ${binaryPath}`, verbose);
if (config.forceVersion) {
if (config.replaceExisting) {
log(`Force version mode: installing specified version ${config.version}`, verbose);
await downloadAndInstallFn(config, verbose);
return binaryPath;
}
if (config.forceVersion) {
log(`Pinned version mode: using installed version ${config.version}`, verbose);
return binaryPath;
}
if (config.skipAutoUpdate) {
log('Runtime bootstrap mode: skipping auto-update check', verbose);
return binaryPath;
+27 -22
View File
@@ -19,6 +19,7 @@ import {
clearPinnedVersion,
isVersionPinned,
resolveLocalBackend,
withCliproxyInstallLifecycleLock,
} from '../binary-manager';
import { BACKEND_CONFIG, DEFAULT_BACKEND } from '../binary/platform-detector';
import { CLIProxyBackend } from '../types';
@@ -127,14 +128,16 @@ export async function installVersion(
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
try {
await installCliproxyVersion(version, verbose, effectiveBackend);
savePinnedVersion(version, effectiveBackend);
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
await installCliproxyVersion(version, verbose, effectiveBackend);
savePinnedVersion(version, effectiveBackend);
return {
success: true,
version,
wasPinned: true,
};
return {
success: true,
version,
wasPinned: true,
};
});
} catch (error) {
return {
success: false,
@@ -156,26 +159,28 @@ export async function installLatest(
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
try {
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
const wasPinned = isVersionPinned(effectiveBackend);
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
const wasPinned = isVersionPinned(effectiveBackend);
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
return {
success: true,
version: latestVersion,
error: `Already running latest version: v${latestVersion}`,
};
}
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
clearPinnedVersion(effectiveBackend);
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
return {
success: true,
version: latestVersion,
error: `Already running latest version: v${latestVersion}`,
wasPinned,
};
}
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
clearPinnedVersion(effectiveBackend);
return {
success: true,
version: latestVersion,
wasPinned,
};
});
} catch (error) {
return {
success: false,
@@ -56,6 +56,7 @@ describe('types.ts backward compatibility', () => {
maxRetries: 3,
verbose: false,
forceVersion: false,
replaceExisting: false,
skipAutoUpdate: false,
allowInstall: true,
};
+1
View File
@@ -14,6 +14,7 @@ export interface BinaryManagerConfig {
maxRetries: number;
verbose: boolean;
forceVersion: boolean;
replaceExisting?: boolean;
skipAutoUpdate: boolean;
allowInstall: boolean;
backend?: CLIProxyBackend;
@@ -1,4 +1,8 @@
import { installCliproxyVersion, resolveLocalBackend } from '../../cliproxy/binary-manager';
import {
installCliproxyVersion,
resolveLocalBackend,
withCliproxyInstallLifecycleLock,
} from '../../cliproxy/binary-manager';
import { resolveLifecyclePort } from '../../cliproxy/config/port-manager';
import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/service-manager';
import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker';
@@ -25,6 +29,7 @@ interface InstallDashboardCliproxyVersionDeps {
ensureCliproxyService: () => Promise<ServiceStartResult>;
isRunningUnderSupervisord?: () => boolean;
restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord;
withInstallLifecycleLock?: typeof withCliproxyInstallLifecycleLock;
}
const defaultDeps: InstallDashboardCliproxyVersionDeps = {
@@ -75,49 +80,53 @@ export async function installDashboardCliproxyVersion(
): Promise<DashboardCliproxyInstallResult> {
const effectiveBackend = resolveLocalBackend(backend, { notifyOnPlus: true });
const backendLabel = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
const shouldRestoreService = await wasProxyRunning(deps);
const withLifecycleLock = deps.withInstallLifecycleLock ?? withCliproxyInstallLifecycleLock;
// The installer owns the stop-and-replace lifecycle, including best-effort
// shutdown for tracked and untracked proxies before swapping the binary.
try {
await deps.installCliproxyVersion(version, true, effectiveBackend);
} catch (error) {
if (shouldRestoreService) {
const restoreResult = await restoreProxyService(deps);
if (!restoreResult.started && !restoreResult.alreadyRunning) {
const installMessage = error instanceof Error ? error.message : String(error);
throw new ProxyError(
`${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
restoreResult.port
);
return withLifecycleLock(effectiveBackend, async () => {
const shouldRestoreService = await wasProxyRunning(deps);
// The installer owns the stop-and-replace lifecycle, including best-effort
// shutdown for tracked and untracked proxies before swapping the binary.
try {
await deps.installCliproxyVersion(version, true, effectiveBackend);
} catch (error) {
if (shouldRestoreService) {
const restoreResult = await restoreProxyService(deps);
if (!restoreResult.started && !restoreResult.alreadyRunning) {
const installMessage = error instanceof Error ? error.message : String(error);
throw new ProxyError(
`${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
restoreResult.port
);
}
}
throw error;
}
if (!shouldRestoreService) {
return {
success: true,
restarted: false,
message: `Successfully installed ${backendLabel} v${version}`,
};
}
// In Docker, supervisord owns process lifecycle — delegate restart to it
const startResult = await restoreProxyService(deps);
if (!startResult.started && !startResult.alreadyRunning) {
return {
success: false,
restarted: false,
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
};
}
throw error;
}
if (!shouldRestoreService) {
return {
success: true,
restarted: false,
message: `Successfully installed ${backendLabel} v${version}`,
restarted: true,
port: startResult.port,
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
};
}
// In Docker, supervisord owns process lifecycle — delegate restart to it
const startResult = await restoreProxyService(deps);
if (!startResult.started && !startResult.alreadyRunning) {
return {
success: false,
restarted: false,
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
};
}
return {
success: true,
restarted: true,
port: startResult.port,
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
};
});
}
@@ -1,5 +1,8 @@
import { readFileSync } from 'fs';
import { mkdtempSync, readFileSync, rmSync } from 'fs';
import { spawnSync } from 'child_process';
import { describe, expect, it } from 'bun:test';
import { tmpdir } from 'os';
import { join } from 'path';
const updateScript = readFileSync('docker/host/ccs-cliproxy-safe-update.sh', 'utf8');
const reconcileScript = readFileSync('docker/host/ccs-cliproxy-reconcile.sh', 'utf8');
@@ -26,6 +29,54 @@ describe('CLIProxy Docker host continuity assets', () => {
expect(reconcileScript).toContain('flock -n 9');
});
it('shares the in-container install lifecycle lock with CLI and dashboard installs', () => {
expect(updateScript).toContain(
"install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'"
);
expect(updateScript).toContain('install_lock_dir="$install_lock_target.lock"');
expect(updateScript).toContain('while ! mkdir "$install_lock_dir"');
expect(updateScript).toContain('touch "$install_lock_dir"');
expect(updateScript).toContain('install_lock_stale_seconds=600');
expect(updateScript).toContain('remove_stale_install_lock');
expect(updateScript).toContain('rmdir "$install_lock_dir"');
expect(updateScript.indexOf('acquire_install_lock')).toBeLessThan(
updateScript.indexOf('supervisorctl_cmd stop cliproxy')
);
});
it('reclaims an orphaned stale lifecycle lock', () => {
const testRoot = mkdtempSync(join(tmpdir(), 'ccs-cliproxy-host-lock-'));
const helpersStart = updateScript.indexOf('remove_stale_install_lock() {');
const helpersEnd = updateScript.indexOf('\nwait_for_proxy() {');
const helpers = updateScript.slice(helpersStart, helpersEnd);
try {
const result = spawnSync(
'bash',
[
'-c',
`set -Eeuo pipefail
install_lock_target="$1/target"
install_lock_dir="$install_lock_target.lock"
install_lock_stale_seconds=600
install_lock_owned=0
mkdir -p "$install_lock_target" "$install_lock_dir"
touch -t 200001010000 "$install_lock_dir"
${helpers}
acquire_install_lock
test "$install_lock_owned" -eq 1
release_install_lock`,
'test-shell',
testRoot,
],
{ encoding: 'utf8' }
);
expect(result.status).toBe(0);
} finally {
rmSync(testRoot, { recursive: true, force: true });
}
});
it('rolls back failed swaps and health-checks both services', () => {
expect(updateScript).toContain('rollback()');
expect(updateScript).toContain('previous-binary');
@@ -41,6 +92,7 @@ describe('CLIProxy Docker host continuity assets', () => {
expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build');
expect(reconcileScript).toContain('restart ccs-dashboard cliproxy');
expect(reconcileScript).toContain('docker restart "$container_name"');
expect(reconcileScript).toContain('up -d --force-recreate --no-build');
expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan(
reconcileScript.indexOf('restart ccs-dashboard cliproxy')
);
@@ -53,4 +105,36 @@ describe('CLIProxy Docker host continuity assets', () => {
expect(reconcileService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh');
expect(reconcileTimer).toContain('OnUnitActiveSec=30s');
});
it('forces nonzero signal exits so the EXIT trap rolls back maintenance', () => {
expect(updateScript).toContain("trap 'exit 130' INT");
expect(updateScript).toContain("trap 'exit 143' TERM");
expect(updateScript).toContain("trap 'exit 129' HUP");
const handlerStart = updateScript.indexOf('on_exit() {');
const handlerEnd = updateScript.indexOf('\ntrap on_exit EXIT');
const handlers = updateScript.slice(handlerStart, handlerEnd);
const result = spawnSync(
'bash',
[
'-c',
`set -Eeuo pipefail
maintenance_started=1
stage_root=/tmp/unused
rollback() { printf 'rollback\\n'; }
cleanup() { printf 'cleanup\\n'; }
release_install_lock() { :; }
${handlers}
trap on_exit EXIT
trap 'exit 143' TERM
true
kill -TERM $$`,
],
{ encoding: 'utf8' }
);
expect(result.status).toBe(143);
expect(result.stdout).toContain('rollback');
expect(result.stdout).toContain('cleanup');
});
});
@@ -43,6 +43,10 @@ function createDeps(
}
);
},
withInstallLifecycleLock: async (
_backend: CLIProxyBackend,
operation: () => Promise<unknown>
) => operation(),
};
return { deps, calls };
@@ -136,4 +140,55 @@ describe('installDashboardCliproxyVersion', () => {
);
expect(calls.ensureCliproxyService).toBe(1);
});
it('serializes concurrent dashboard stop-install-restore transactions', async () => {
let running = true;
let queue = Promise.resolve();
let activeTransactions = 0;
let maxActiveTransactions = 0;
let restores = 0;
const deps = {
getProxyStatus: () => ({ running }),
isCliproxyRunning: async () => running,
installCliproxyVersion: async () => {
running = false;
await new Promise((resolve) => setTimeout(resolve, 5));
},
ensureCliproxyService: async () => {
running = true;
restores += 1;
return { started: true, alreadyRunning: false, port: 8317 };
},
withInstallLifecycleLock: <T>(
_backend: CLIProxyBackend,
operation: () => Promise<T>
): Promise<T> => {
const result = queue.then(async () => {
activeTransactions += 1;
maxActiveTransactions = Math.max(maxActiveTransactions, activeTransactions);
try {
return await operation();
} finally {
activeTransactions -= 1;
}
});
queue = result.then(
() => undefined,
() => undefined
);
return result;
},
};
const results = await Promise.all([
installDashboardCliproxyVersion('6.7.1', 'plus', deps),
installDashboardCliproxyVersion('6.7.2', 'plus', deps),
]);
expect(maxActiveTransactions).toBe(1);
expect(restores).toBe(2);
expect(results.every((result) => result.restarted)).toBe(true);
expect(running).toBe(true);
});
});