mirror of
https://github.com/tiennm99/ccs.git
synced 2026-10-03 13:12:09 +00:00
fix(cliproxy): finish concurrent update recovery
This commit is contained in:
1 parent
192b27fbb2
commit
fc56ecaac4
14 files changed
+428
-87
No files matched your search
@@ -44,6 +44,14 @@ compose_up() {
|
||||
-f "$compose_file" up -d --build
|
||||
}
|
||||
|
||||
compose_recreate() {
|
||||
cd "$compose_dir"
|
||||
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
|
||||
-f "$compose_file" up -d --force-recreate --no-build || \
|
||||
docker compose --project-name "$compose_project" --project-directory "$compose_dir" \
|
||||
-f "$compose_file" up -d --force-recreate --build
|
||||
}
|
||||
|
||||
if ! docker inspect "$container_name" >/dev/null 2>&1; then
|
||||
log "Container $container_name is missing; recreating from $compose_file"
|
||||
compose_up
|
||||
@@ -79,6 +87,14 @@ if docker exec "$container_name" \
|
||||
fi
|
||||
|
||||
log 'Supervisor recovery failed; restarting the container'
|
||||
docker restart "$container_name" >/dev/null
|
||||
if docker restart "$container_name" >/dev/null; then
|
||||
if wait_for_health; then
|
||||
log "Container $container_name recovered and passed both health probes"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Container $container_name is still unhealthy; recreating it from $compose_file"
|
||||
compose_recreate
|
||||
wait_for_health
|
||||
log "Container $container_name recovered and passed both health probes"
|
||||
log "Container $container_name was recreated and passed both health probes"
|
||||
@@ -55,6 +55,10 @@ stage_binary="$stage_dir/cli-proxy-api-plus"
|
||||
stage_version="$stage_dir/.version"
|
||||
backup_binary="$stage_root/previous-binary"
|
||||
backup_version="$stage_root/previous-version"
|
||||
install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'
|
||||
install_lock_dir="$install_lock_target.lock"
|
||||
install_lock_stale_seconds=600
|
||||
install_lock_owned=0
|
||||
maintenance_started=0
|
||||
|
||||
supervisorctl_cmd() {
|
||||
@@ -65,6 +69,40 @@ cleanup() {
|
||||
rm -rf -- "$stage_root"
|
||||
}
|
||||
|
||||
remove_stale_install_lock() {
|
||||
lock_mtime="$(
|
||||
stat -c %Y "$install_lock_dir" 2>/dev/null || stat -f %m "$install_lock_dir" 2>/dev/null
|
||||
)" || return 0
|
||||
current_time="$(date +%s)"
|
||||
lock_age=$((current_time - lock_mtime))
|
||||
if [ "$lock_age" -gt "$install_lock_stale_seconds" ]; then
|
||||
rmdir "$install_lock_dir" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_install_lock() {
|
||||
mkdir -p "$install_lock_target"
|
||||
attempts=0
|
||||
while ! mkdir "$install_lock_dir" 2>/dev/null; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 240 ]; then
|
||||
printf '[X] Timed out waiting for CLIProxy install lifecycle lock\n' >&2
|
||||
return 1
|
||||
fi
|
||||
remove_stale_install_lock
|
||||
sleep 0.25
|
||||
done
|
||||
install_lock_owned=1
|
||||
touch "$install_lock_dir"
|
||||
}
|
||||
|
||||
release_install_lock() {
|
||||
if [ "$install_lock_owned" -eq 1 ]; then
|
||||
rmdir "$install_lock_dir" 2>/dev/null || true
|
||||
install_lock_owned=0
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_proxy() {
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
@@ -107,6 +145,7 @@ on_exit() {
|
||||
printf '[X] CLIProxy rollback failed; recovery files preserved at %s\n' "$stage_root" >&2
|
||||
fi
|
||||
fi
|
||||
release_install_lock
|
||||
if [ "$rollback_failed" -eq 0 ]; then
|
||||
cleanup
|
||||
else
|
||||
@@ -115,7 +154,10 @@ on_exit() {
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
trap on_exit EXIT INT TERM HUP
|
||||
trap on_exit EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
trap 'exit 129' HUP
|
||||
|
||||
mkdir -p "$stage_ccs_dir"
|
||||
CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest --backend plus
|
||||
@@ -123,6 +165,7 @@ test -x "$stage_binary"
|
||||
test -s "$stage_version"
|
||||
"$stage_binary" --version >/dev/null
|
||||
|
||||
acquire_install_lock
|
||||
cp -p "$live_binary" "$backup_binary"
|
||||
cp -p "$live_version" "$backup_version"
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
import type { CLIProxyBackend } from './types';
|
||||
import { getVersionListCachePath } from './binary/version-cache';
|
||||
import { loadOrCreateUnifiedConfig } from '../config/config-loader-facade';
|
||||
import { withInstallLifecycleLock } from './binary/install-lifecycle-lock';
|
||||
|
||||
export const CLIPROXY_DELETED_PLUS_REPO = 'router-for-me/CLIProxyAPIPlus';
|
||||
export const CLIPROXY_PLUS_FALLBACK_TRACKING_URL = 'https://github.com/kaitranntt/ccs/issues/1062';
|
||||
@@ -176,6 +177,14 @@ function getBackendBinDir(backend: CLIProxyBackend = DEFAULT_BACKEND): string {
|
||||
return `${baseDir}/${backend}`;
|
||||
}
|
||||
|
||||
export async function withCliproxyInstallLifecycleLock<T>(
|
||||
backend: CLIProxyBackend,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const lockTarget = path.join(getBinDir(), `.install-lifecycle-${backend}`);
|
||||
return withInstallLifecycleLock(lockTarget, operation);
|
||||
}
|
||||
|
||||
/** Default configuration (uses backend from config.yaml or defaults to `DEFAULT_BACKEND`) */
|
||||
function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): BinaryManagerConfig {
|
||||
const backendConfig = BACKEND_CONFIG[backend];
|
||||
@@ -186,6 +195,7 @@ function createDefaultConfig(backend: CLIProxyBackend = DEFAULT_BACKEND): Binary
|
||||
maxRetries: 3,
|
||||
verbose: false,
|
||||
forceVersion: false,
|
||||
replaceExisting: false,
|
||||
skipAutoUpdate: false,
|
||||
allowInstall: true,
|
||||
backend, // Pass backend for installer to use correct download URL
|
||||
@@ -317,6 +327,7 @@ interface InstallCliproxyVersionDeps {
|
||||
formatInfo?: typeof info;
|
||||
formatWarn?: typeof warn;
|
||||
getInstalledVersion?: typeof getInstalledCliproxyVersion;
|
||||
withInstallLifecycleLockFn?: typeof withCliproxyInstallLifecycleLock;
|
||||
}
|
||||
|
||||
/** Install a specific version of CLIProxyAPI */
|
||||
@@ -329,33 +340,41 @@ export async function installCliproxyVersion(
|
||||
const configuredBackend = backend ?? getConfiguredOrDefaultBackend();
|
||||
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
|
||||
const manager =
|
||||
deps.createManager?.({ version, verbose, forceVersion: true }, effectiveBackend) ??
|
||||
new BinaryManager({ version, verbose, forceVersion: true }, effectiveBackend);
|
||||
deps.createManager?.(
|
||||
{ version, verbose, forceVersion: true, replaceExisting: true },
|
||||
effectiveBackend
|
||||
) ??
|
||||
new BinaryManager(
|
||||
{ version, verbose, forceVersion: true, replaceExisting: true },
|
||||
effectiveBackend
|
||||
);
|
||||
const stopProxyFn = deps.stopProxyFn ?? stopProxy;
|
||||
const waitForPortFreeFn = deps.waitForPortFreeFn ?? waitForPortFree;
|
||||
const formatInfo = deps.formatInfo ?? info;
|
||||
const formatWarn = deps.formatWarn ?? warn;
|
||||
const withLifecycleLock = deps.withInstallLifecycleLockFn ?? withCliproxyInstallLifecycleLock;
|
||||
|
||||
// Always attempt a best-effort stop first so we also catch untracked proxies
|
||||
// that are running without a session lock.
|
||||
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
|
||||
const result = await stopProxyFn();
|
||||
if (result.stopped) {
|
||||
const stoppedPort = result.port ?? resolveLifecyclePort();
|
||||
// Wait for port to be fully released
|
||||
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
|
||||
if (!portFree && verbose) {
|
||||
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
|
||||
await withLifecycleLock(effectiveBackend, async () => {
|
||||
// Always attempt a best-effort stop first so we also catch untracked proxies
|
||||
// that are running without a session lock.
|
||||
if (verbose) console.log(formatInfo('Stopping running CLIProxy before update...'));
|
||||
const result = await stopProxyFn();
|
||||
if (result.stopped) {
|
||||
const stoppedPort = result.port ?? resolveLifecyclePort();
|
||||
const portFree = await waitForPortFreeFn(stoppedPort, 5000);
|
||||
if (!portFree && verbose) {
|
||||
console.log(formatWarn('Port did not free up in time, proceeding anyway...'));
|
||||
}
|
||||
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
|
||||
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
|
||||
}
|
||||
} else if (verbose && result.error && result.error !== 'No active CLIProxy session found') {
|
||||
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
|
||||
}
|
||||
|
||||
await manager.ensureBinary();
|
||||
await manager.ensureBinary();
|
||||
|
||||
if (verbose) {
|
||||
console.log(formatInfo('New version will be active on next CLIProxy command'));
|
||||
}
|
||||
if (verbose) {
|
||||
console.log(formatInfo('New version will be active on next CLIProxy command'));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Fetch the latest CLIProxyAPI version from GitHub API */
|
||||
|
||||
@@ -5,6 +5,7 @@ import * as path from 'path';
|
||||
import { getExecutableName } from '../platform-detector';
|
||||
import { downloadAndInstall } from '../installer';
|
||||
import { ensureBinary } from '../lifecycle';
|
||||
import { withInstallLifecycleLock } from '../install-lifecycle-lock';
|
||||
|
||||
describe('atomic binary installation', () => {
|
||||
let binPath: string;
|
||||
@@ -78,6 +79,7 @@ describe('atomic binary installation', () => {
|
||||
maxRetries: 1,
|
||||
verbose: false,
|
||||
forceVersion: true,
|
||||
replaceExisting: true,
|
||||
skipAutoUpdate: false,
|
||||
allowInstall: true,
|
||||
backend: 'original',
|
||||
@@ -163,6 +165,7 @@ describe('atomic binary installation', () => {
|
||||
maxRetries: 1,
|
||||
verbose: false,
|
||||
forceVersion: true,
|
||||
replaceExisting: true,
|
||||
skipAutoUpdate: false,
|
||||
allowInstall: true,
|
||||
backend: 'original',
|
||||
@@ -178,6 +181,52 @@ describe('atomic binary installation', () => {
|
||||
expect(installs).toBe(1);
|
||||
});
|
||||
|
||||
it('reuses an existing pinned binary during runtime bootstrap', async () => {
|
||||
const binaryPath = path.join(binPath, getExecutableName('original'));
|
||||
fs.writeFileSync(binaryPath, 'pinned-binary');
|
||||
let installs = 0;
|
||||
|
||||
const resolvedPath = await ensureBinary(
|
||||
{
|
||||
version: '6.6.80',
|
||||
releaseUrl: 'https://example.invalid',
|
||||
binPath,
|
||||
maxRetries: 1,
|
||||
verbose: false,
|
||||
forceVersion: true,
|
||||
replaceExisting: false,
|
||||
skipAutoUpdate: false,
|
||||
allowInstall: false,
|
||||
backend: 'original',
|
||||
},
|
||||
{
|
||||
downloadAndInstallFn: async () => {
|
||||
installs += 1;
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
expect(resolvedPath).toBe(binaryPath);
|
||||
expect(installs).toBe(0);
|
||||
});
|
||||
|
||||
it('waits for an externally held compatible install lifecycle lock', async () => {
|
||||
const lockTarget = path.join(binPath, '.install-lifecycle-plus');
|
||||
fs.mkdirSync(lockTarget, { recursive: true });
|
||||
fs.mkdirSync(`${lockTarget}.lock`);
|
||||
let entered = false;
|
||||
|
||||
const operation = withInstallLifecycleLock(lockTarget, async () => {
|
||||
entered = true;
|
||||
});
|
||||
|
||||
await Bun.sleep(50);
|
||||
expect(entered).toBe(false);
|
||||
fs.rmdirSync(`${lockTarget}.lock`);
|
||||
await operation;
|
||||
expect(entered).toBe(true);
|
||||
});
|
||||
|
||||
it('restores the previous binary when publishing the version marker fails', async () => {
|
||||
const binaryName = getExecutableName('original');
|
||||
const binaryPath = path.join(binPath, binaryName);
|
||||
|
||||
@@ -33,6 +33,7 @@ describe('installCliproxyVersion', () => {
|
||||
);
|
||||
|
||||
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
|
||||
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
|
||||
createManager: (_config: unknown, backend: string) => {
|
||||
seenBackend = backend;
|
||||
return {
|
||||
@@ -147,6 +148,7 @@ describe('installCliproxyVersion', () => {
|
||||
const binaryManager = await import(`../../binary-manager?binary-manager-install=${Date.now()}`);
|
||||
|
||||
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
|
||||
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
|
||||
createManager: () => ({
|
||||
isBinaryInstalled: () => false,
|
||||
deleteBinary: () => {
|
||||
@@ -187,6 +189,7 @@ describe('installCliproxyVersion', () => {
|
||||
);
|
||||
|
||||
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
|
||||
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
|
||||
createManager: () => ({
|
||||
isBinaryInstalled: () => true,
|
||||
deleteBinary: () => {
|
||||
@@ -216,6 +219,7 @@ describe('installCliproxyVersion', () => {
|
||||
);
|
||||
|
||||
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
|
||||
withInstallLifecycleLockFn: async (_backend, operation) => operation(),
|
||||
createManager: () => ({
|
||||
isBinaryInstalled: () => false,
|
||||
deleteBinary: () => undefined,
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { AsyncLocalStorage } from 'async_hooks';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as lockfile from 'proper-lockfile';
|
||||
|
||||
const heldInstallLocks = new AsyncLocalStorage<Set<string>>();
|
||||
|
||||
export async function withInstallLifecycleLock<T>(
|
||||
lockTarget: string,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const resolvedTarget = path.resolve(lockTarget);
|
||||
const held = heldInstallLocks.getStore();
|
||||
if (held?.has(resolvedTarget)) return operation();
|
||||
|
||||
fs.mkdirSync(resolvedTarget, { recursive: true });
|
||||
const release = await lockfile.lock(resolvedTarget, {
|
||||
stale: 10 * 60 * 1000,
|
||||
retries: { retries: 60, factor: 1, minTimeout: 250, maxTimeout: 250 },
|
||||
});
|
||||
const nextHeld = new Set(held);
|
||||
nextHeld.add(resolvedTarget);
|
||||
let operationError: unknown;
|
||||
|
||||
try {
|
||||
return await heldInstallLocks.run(nextHeld, operation);
|
||||
} catch (error) {
|
||||
operationError = error;
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
await release();
|
||||
} catch (error) {
|
||||
if (!operationError) throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,6 +53,7 @@ export async function downloadAndInstall(
|
||||
});
|
||||
let stagingPath: string | undefined;
|
||||
const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`);
|
||||
let installError: unknown;
|
||||
|
||||
try {
|
||||
for (const entry of fs.readdirSync(config.binPath)) {
|
||||
@@ -130,11 +131,23 @@ export async function downloadAndInstall(
|
||||
spinner.succeed(`${backendLabel} ready`);
|
||||
console.log(ok(`${backendLabel} v${config.version} installed successfully`));
|
||||
} catch (error) {
|
||||
installError = error;
|
||||
spinner.fail('Installation failed');
|
||||
throw error;
|
||||
} finally {
|
||||
if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
|
||||
await releaseLock();
|
||||
let cleanupError: unknown;
|
||||
try {
|
||||
if (stagingPath) fs.rmSync(stagingPath, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
} finally {
|
||||
try {
|
||||
await releaseLock();
|
||||
} catch (error) {
|
||||
cleanupError ??= error;
|
||||
}
|
||||
}
|
||||
if (!installError && cleanupError) throw cleanupError;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -121,12 +121,17 @@ export async function ensureBinary(
|
||||
if (fs.existsSync(binaryPath)) {
|
||||
log(`Binary exists: ${binaryPath}`, verbose);
|
||||
|
||||
if (config.forceVersion) {
|
||||
if (config.replaceExisting) {
|
||||
log(`Force version mode: installing specified version ${config.version}`, verbose);
|
||||
await downloadAndInstallFn(config, verbose);
|
||||
return binaryPath;
|
||||
}
|
||||
|
||||
if (config.forceVersion) {
|
||||
log(`Pinned version mode: using installed version ${config.version}`, verbose);
|
||||
return binaryPath;
|
||||
}
|
||||
|
||||
if (config.skipAutoUpdate) {
|
||||
log('Runtime bootstrap mode: skipping auto-update check', verbose);
|
||||
return binaryPath;
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
clearPinnedVersion,
|
||||
isVersionPinned,
|
||||
resolveLocalBackend,
|
||||
withCliproxyInstallLifecycleLock,
|
||||
} from '../binary-manager';
|
||||
import { BACKEND_CONFIG, DEFAULT_BACKEND } from '../binary/platform-detector';
|
||||
import { CLIProxyBackend } from '../types';
|
||||
@@ -127,14 +128,16 @@ export async function installVersion(
|
||||
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
|
||||
|
||||
try {
|
||||
await installCliproxyVersion(version, verbose, effectiveBackend);
|
||||
savePinnedVersion(version, effectiveBackend);
|
||||
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
|
||||
await installCliproxyVersion(version, verbose, effectiveBackend);
|
||||
savePinnedVersion(version, effectiveBackend);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
version,
|
||||
wasPinned: true,
|
||||
};
|
||||
return {
|
||||
success: true,
|
||||
version,
|
||||
wasPinned: true,
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
@@ -156,26 +159,28 @@ export async function installLatest(
|
||||
const effectiveBackend = resolveLocalBackend(configuredBackend, { notifyOnPlus: true });
|
||||
|
||||
try {
|
||||
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
|
||||
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
|
||||
const wasPinned = isVersionPinned(effectiveBackend);
|
||||
return await withCliproxyInstallLifecycleLock(effectiveBackend, async () => {
|
||||
const latestVersion = await fetchLatestCliproxyVersion(effectiveBackend);
|
||||
const currentVersion = getInstalledCliproxyVersion(effectiveBackend);
|
||||
const wasPinned = isVersionPinned(effectiveBackend);
|
||||
|
||||
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
|
||||
return {
|
||||
success: true,
|
||||
version: latestVersion,
|
||||
error: `Already running latest version: v${latestVersion}`,
|
||||
};
|
||||
}
|
||||
|
||||
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
|
||||
clearPinnedVersion(effectiveBackend);
|
||||
|
||||
if (isCLIProxyInstalled(effectiveBackend) && latestVersion === currentVersion && !wasPinned) {
|
||||
return {
|
||||
success: true,
|
||||
version: latestVersion,
|
||||
error: `Already running latest version: v${latestVersion}`,
|
||||
wasPinned,
|
||||
};
|
||||
}
|
||||
|
||||
await installCliproxyVersion(latestVersion, verbose, effectiveBackend);
|
||||
clearPinnedVersion(effectiveBackend);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
version: latestVersion,
|
||||
wasPinned,
|
||||
};
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
|
||||
@@ -56,6 +56,7 @@ describe('types.ts backward compatibility', () => {
|
||||
maxRetries: 3,
|
||||
verbose: false,
|
||||
forceVersion: false,
|
||||
replaceExisting: false,
|
||||
skipAutoUpdate: false,
|
||||
allowInstall: true,
|
||||
};
|
||||
|
||||
@@ -14,6 +14,7 @@ export interface BinaryManagerConfig {
|
||||
maxRetries: number;
|
||||
verbose: boolean;
|
||||
forceVersion: boolean;
|
||||
replaceExisting?: boolean;
|
||||
skipAutoUpdate: boolean;
|
||||
allowInstall: boolean;
|
||||
backend?: CLIProxyBackend;
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { installCliproxyVersion, resolveLocalBackend } from '../../cliproxy/binary-manager';
|
||||
import {
|
||||
installCliproxyVersion,
|
||||
resolveLocalBackend,
|
||||
withCliproxyInstallLifecycleLock,
|
||||
} from '../../cliproxy/binary-manager';
|
||||
import { resolveLifecyclePort } from '../../cliproxy/config/port-manager';
|
||||
import { ensureCliproxyService, type ServiceStartResult } from '../../cliproxy/service-manager';
|
||||
import { getProxyStatus as getProxyProcessStatus } from '../../cliproxy/session-tracker';
|
||||
@@ -25,6 +29,7 @@ interface InstallDashboardCliproxyVersionDeps {
|
||||
ensureCliproxyService: () => Promise<ServiceStartResult>;
|
||||
isRunningUnderSupervisord?: () => boolean;
|
||||
restartCliproxyViaSupervisord?: typeof restartCliproxyViaSupervisord;
|
||||
withInstallLifecycleLock?: typeof withCliproxyInstallLifecycleLock;
|
||||
}
|
||||
|
||||
const defaultDeps: InstallDashboardCliproxyVersionDeps = {
|
||||
@@ -75,49 +80,53 @@ export async function installDashboardCliproxyVersion(
|
||||
): Promise<DashboardCliproxyInstallResult> {
|
||||
const effectiveBackend = resolveLocalBackend(backend, { notifyOnPlus: true });
|
||||
const backendLabel = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
|
||||
const shouldRestoreService = await wasProxyRunning(deps);
|
||||
const withLifecycleLock = deps.withInstallLifecycleLock ?? withCliproxyInstallLifecycleLock;
|
||||
|
||||
// The installer owns the stop-and-replace lifecycle, including best-effort
|
||||
// shutdown for tracked and untracked proxies before swapping the binary.
|
||||
try {
|
||||
await deps.installCliproxyVersion(version, true, effectiveBackend);
|
||||
} catch (error) {
|
||||
if (shouldRestoreService) {
|
||||
const restoreResult = await restoreProxyService(deps);
|
||||
if (!restoreResult.started && !restoreResult.alreadyRunning) {
|
||||
const installMessage = error instanceof Error ? error.message : String(error);
|
||||
throw new ProxyError(
|
||||
`${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
|
||||
restoreResult.port
|
||||
);
|
||||
return withLifecycleLock(effectiveBackend, async () => {
|
||||
const shouldRestoreService = await wasProxyRunning(deps);
|
||||
|
||||
// The installer owns the stop-and-replace lifecycle, including best-effort
|
||||
// shutdown for tracked and untracked proxies before swapping the binary.
|
||||
try {
|
||||
await deps.installCliproxyVersion(version, true, effectiveBackend);
|
||||
} catch (error) {
|
||||
if (shouldRestoreService) {
|
||||
const restoreResult = await restoreProxyService(deps);
|
||||
if (!restoreResult.started && !restoreResult.alreadyRunning) {
|
||||
const installMessage = error instanceof Error ? error.message : String(error);
|
||||
throw new ProxyError(
|
||||
`${installMessage}; previous ${backendLabel} service also failed to restart: ${restoreResult.error ?? 'unknown restart error'}`,
|
||||
restoreResult.port
|
||||
);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!shouldRestoreService) {
|
||||
return {
|
||||
success: true,
|
||||
restarted: false,
|
||||
message: `Successfully installed ${backendLabel} v${version}`,
|
||||
};
|
||||
}
|
||||
|
||||
// In Docker, supervisord owns process lifecycle — delegate restart to it
|
||||
const startResult = await restoreProxyService(deps);
|
||||
if (!startResult.started && !startResult.alreadyRunning) {
|
||||
return {
|
||||
success: false,
|
||||
restarted: false,
|
||||
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
|
||||
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!shouldRestoreService) {
|
||||
return {
|
||||
success: true,
|
||||
restarted: false,
|
||||
message: `Successfully installed ${backendLabel} v${version}`,
|
||||
restarted: true,
|
||||
port: startResult.port,
|
||||
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
|
||||
};
|
||||
}
|
||||
|
||||
// In Docker, supervisord owns process lifecycle — delegate restart to it
|
||||
const startResult = await restoreProxyService(deps);
|
||||
if (!startResult.started && !startResult.alreadyRunning) {
|
||||
return {
|
||||
success: false,
|
||||
restarted: false,
|
||||
error: startResult.error || `Installed ${backendLabel} v${version}, but restart failed`,
|
||||
message: `Installed ${backendLabel} v${version}, but failed to restart it`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
restarted: true,
|
||||
port: startResult.port,
|
||||
message: `Successfully installed ${backendLabel} v${version} and restarted it on port ${startResult.port}`,
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'fs';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
const updateScript = readFileSync('docker/host/ccs-cliproxy-safe-update.sh', 'utf8');
|
||||
const reconcileScript = readFileSync('docker/host/ccs-cliproxy-reconcile.sh', 'utf8');
|
||||
@@ -26,6 +29,54 @@ describe('CLIProxy Docker host continuity assets', () => {
|
||||
expect(reconcileScript).toContain('flock -n 9');
|
||||
});
|
||||
|
||||
it('shares the in-container install lifecycle lock with CLI and dashboard installs', () => {
|
||||
expect(updateScript).toContain(
|
||||
"install_lock_target='/root/.ccs/cliproxy/bin/.install-lifecycle-plus'"
|
||||
);
|
||||
expect(updateScript).toContain('install_lock_dir="$install_lock_target.lock"');
|
||||
expect(updateScript).toContain('while ! mkdir "$install_lock_dir"');
|
||||
expect(updateScript).toContain('touch "$install_lock_dir"');
|
||||
expect(updateScript).toContain('install_lock_stale_seconds=600');
|
||||
expect(updateScript).toContain('remove_stale_install_lock');
|
||||
expect(updateScript).toContain('rmdir "$install_lock_dir"');
|
||||
expect(updateScript.indexOf('acquire_install_lock')).toBeLessThan(
|
||||
updateScript.indexOf('supervisorctl_cmd stop cliproxy')
|
||||
);
|
||||
});
|
||||
|
||||
it('reclaims an orphaned stale lifecycle lock', () => {
|
||||
const testRoot = mkdtempSync(join(tmpdir(), 'ccs-cliproxy-host-lock-'));
|
||||
const helpersStart = updateScript.indexOf('remove_stale_install_lock() {');
|
||||
const helpersEnd = updateScript.indexOf('\nwait_for_proxy() {');
|
||||
const helpers = updateScript.slice(helpersStart, helpersEnd);
|
||||
try {
|
||||
const result = spawnSync(
|
||||
'bash',
|
||||
[
|
||||
'-c',
|
||||
`set -Eeuo pipefail
|
||||
install_lock_target="$1/target"
|
||||
install_lock_dir="$install_lock_target.lock"
|
||||
install_lock_stale_seconds=600
|
||||
install_lock_owned=0
|
||||
mkdir -p "$install_lock_target" "$install_lock_dir"
|
||||
touch -t 200001010000 "$install_lock_dir"
|
||||
${helpers}
|
||||
acquire_install_lock
|
||||
test "$install_lock_owned" -eq 1
|
||||
release_install_lock`,
|
||||
'test-shell',
|
||||
testRoot,
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
);
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
} finally {
|
||||
rmSync(testRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rolls back failed swaps and health-checks both services', () => {
|
||||
expect(updateScript).toContain('rollback()');
|
||||
expect(updateScript).toContain('previous-binary');
|
||||
@@ -41,6 +92,7 @@ describe('CLIProxy Docker host continuity assets', () => {
|
||||
expect(reconcileScript).toContain('-f "$compose_file" up -d --no-build');
|
||||
expect(reconcileScript).toContain('restart ccs-dashboard cliproxy');
|
||||
expect(reconcileScript).toContain('docker restart "$container_name"');
|
||||
expect(reconcileScript).toContain('up -d --force-recreate --no-build');
|
||||
expect(reconcileScript.indexOf('docker restart "$container_name"')).toBeGreaterThan(
|
||||
reconcileScript.indexOf('restart ccs-dashboard cliproxy')
|
||||
);
|
||||
@@ -53,4 +105,36 @@ describe('CLIProxy Docker host continuity assets', () => {
|
||||
expect(reconcileService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh');
|
||||
expect(reconcileTimer).toContain('OnUnitActiveSec=30s');
|
||||
});
|
||||
|
||||
it('forces nonzero signal exits so the EXIT trap rolls back maintenance', () => {
|
||||
expect(updateScript).toContain("trap 'exit 130' INT");
|
||||
expect(updateScript).toContain("trap 'exit 143' TERM");
|
||||
expect(updateScript).toContain("trap 'exit 129' HUP");
|
||||
|
||||
const handlerStart = updateScript.indexOf('on_exit() {');
|
||||
const handlerEnd = updateScript.indexOf('\ntrap on_exit EXIT');
|
||||
const handlers = updateScript.slice(handlerStart, handlerEnd);
|
||||
const result = spawnSync(
|
||||
'bash',
|
||||
[
|
||||
'-c',
|
||||
`set -Eeuo pipefail
|
||||
maintenance_started=1
|
||||
stage_root=/tmp/unused
|
||||
rollback() { printf 'rollback\\n'; }
|
||||
cleanup() { printf 'cleanup\\n'; }
|
||||
release_install_lock() { :; }
|
||||
${handlers}
|
||||
trap on_exit EXIT
|
||||
trap 'exit 143' TERM
|
||||
true
|
||||
kill -TERM $$`,
|
||||
],
|
||||
{ encoding: 'utf8' }
|
||||
);
|
||||
|
||||
expect(result.status).toBe(143);
|
||||
expect(result.stdout).toContain('rollback');
|
||||
expect(result.stdout).toContain('cleanup');
|
||||
});
|
||||
});
|
||||
@@ -43,6 +43,10 @@ function createDeps(
|
||||
}
|
||||
);
|
||||
},
|
||||
withInstallLifecycleLock: async (
|
||||
_backend: CLIProxyBackend,
|
||||
operation: () => Promise<unknown>
|
||||
) => operation(),
|
||||
};
|
||||
|
||||
return { deps, calls };
|
||||
@@ -136,4 +140,55 @@ describe('installDashboardCliproxyVersion', () => {
|
||||
);
|
||||
expect(calls.ensureCliproxyService).toBe(1);
|
||||
});
|
||||
|
||||
it('serializes concurrent dashboard stop-install-restore transactions', async () => {
|
||||
let running = true;
|
||||
let queue = Promise.resolve();
|
||||
let activeTransactions = 0;
|
||||
let maxActiveTransactions = 0;
|
||||
let restores = 0;
|
||||
|
||||
const deps = {
|
||||
getProxyStatus: () => ({ running }),
|
||||
isCliproxyRunning: async () => running,
|
||||
installCliproxyVersion: async () => {
|
||||
running = false;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
},
|
||||
ensureCliproxyService: async () => {
|
||||
running = true;
|
||||
restores += 1;
|
||||
return { started: true, alreadyRunning: false, port: 8317 };
|
||||
},
|
||||
withInstallLifecycleLock: <T>(
|
||||
_backend: CLIProxyBackend,
|
||||
operation: () => Promise<T>
|
||||
): Promise<T> => {
|
||||
const result = queue.then(async () => {
|
||||
activeTransactions += 1;
|
||||
maxActiveTransactions = Math.max(maxActiveTransactions, activeTransactions);
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
activeTransactions -= 1;
|
||||
}
|
||||
});
|
||||
queue = result.then(
|
||||
() => undefined,
|
||||
() => undefined
|
||||
);
|
||||
return result;
|
||||
},
|
||||
};
|
||||
|
||||
const results = await Promise.all([
|
||||
installDashboardCliproxyVersion('6.7.1', 'plus', deps),
|
||||
installDashboardCliproxyVersion('6.7.2', 'plus', deps),
|
||||
]);
|
||||
|
||||
expect(maxActiveTransactions).toBe(1);
|
||||
expect(restores).toBe(2);
|
||||
expect(results.every((result) => result.restarted)).toBe(true);
|
||||
expect(running).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user