fix(cliproxy): keep proxy available during updates

This commit is contained in:
Tam Nhu Tran committed 2026-08-10 22:13:29 -04:00
1 parent 51afccf504
commit 8e4def4713
12 files changed
+521 -54

No files matched your search

+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
set -Eeuo pipefail
container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
compose_dir="${CCS_CLIPROXY_COMPOSE_DIR:-/root/.ccs/docker}"
compose_file="${CCS_CLIPROXY_COMPOSE_FILE:-$compose_dir/docker-compose.integrated.yml}"
lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
log_file="${CCS_CLIPROXY_RECONCILE_LOG:-/var/log/ccs-cliproxy-reconcile.log}"
log() {
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$log_file"
}
exec 9>"$lock_file"
if ! flock -n 9; then
exit 0
fi
probe() {
docker exec "$container_name" sh -c \
'curl -fsS --max-time 2 http://127.0.0.1:3000/ >/dev/null && curl -fsS --max-time 2 http://127.0.0.1:8317/ >/dev/null' \
>/dev/null 2>&1
}
wait_for_health() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if probe; then
return 0
fi
attempts=$((attempts + 1))
sleep 2
done
return 1
}
compose_up() {
cd "$compose_dir"
docker compose -f "$compose_file" up -d --no-build || \
docker compose -f "$compose_file" up -d --build
}
if ! docker inspect "$container_name" >/dev/null 2>&1; then
log "Container $container_name is missing; recreating from $compose_file"
compose_up
wait_for_health
log "Container $container_name was recreated and passed both health probes"
exit 0
fi
if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'true' ]; then
log "Container $container_name is stopped; restoring the Compose service"
compose_up
wait_for_health
log "Container $container_name is running and passed both health probes"
exit 0
fi
if probe; then
exit 0
fi
sleep 10
if probe; then
exit 0
fi
log 'Dashboard or proxy failed two consecutive probes; restarting supervised processes'
docker exec "$container_name" supervisorctl -c /etc/supervisord.conf restart ccs-dashboard cliproxy
if wait_for_health; then
log 'Supervised processes recovered and passed both health probes'
exit 0
fi
log 'Supervisor recovery failed; restarting the container'
docker restart "$container_name" >/dev/null
wait_for_health
log "Container $container_name recovered and passed both health probes"
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
set -Eeuo pipefail
container_name="${CCS_CLIPROXY_CONTAINER:-ccs-cliproxy}"
lock_file="${CCS_CLIPROXY_LOCK_FILE:-/run/lock/ccs-cliproxy-maintenance.lock}"
log_file="${CCS_CLIPROXY_UPDATE_LOG:-/var/log/ccs-cliproxy-update.log}"
log() {
printf '[%s] %s\n' "$(date -Is)" "$*" | tee -a "$log_file"
}
exec 9>"$lock_file"
if ! flock -n 9; then
log 'Another CLIProxy maintenance operation is active; skipping update check'
exit 0
fi
if ! docker inspect "$container_name" >/dev/null 2>&1; then
log "Container $container_name is missing; reconciliation must restore it before updating"
exit 1
fi
if [ "$(docker inspect --format '{{.State.Running}}' "$container_name")" != 'true' ]; then
log "Container $container_name is not running; reconciliation must restore it before updating"
exit 1
fi
status_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)" || {
log "Unable to inspect the installed CLIProxy version: $status_output"
exit 1
}
if ! grep -qi 'update available' <<<"$status_output"; then
exit 0
fi
log 'CLIProxy Plus update available; staging verified replacement while the current proxy stays online'
if ! docker exec -i "$container_name" sh -s <<'CONTAINER_UPDATE'
set -eu
live_dir='/root/.ccs/cliproxy/bin/plus'
live_binary="$live_dir/cli-proxy-api-plus"
live_version="$live_dir/.version"
stage_root="$(mktemp -d /root/.ccs/cliproxy/.host-update.XXXXXX)"
stage_ccs_dir="$stage_root/ccs"
stage_dir="$stage_ccs_dir/cliproxy/bin/plus"
stage_binary="$stage_dir/cli-proxy-api-plus"
stage_version="$stage_dir/.version"
backup_binary="$stage_root/previous-binary"
backup_version="$stage_root/previous-version"
swap_started=0
supervisorctl_cmd() {
supervisorctl -c /etc/supervisord.conf "$@"
}
cleanup() {
rm -rf -- "$stage_root"
}
wait_for_proxy() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if curl -fsS --max-time 2 http://127.0.0.1:8317/ >/dev/null; then
return 0
fi
attempts=$((attempts + 1))
sleep 2
done
return 1
}
rollback() {
supervisorctl_cmd stop cliproxy >/dev/null 2>&1 || true
if [ -f "$backup_binary" ]; then
install -m 0755 "$backup_binary" "$live_binary.rollback"
mv -f "$live_binary.rollback" "$live_binary"
fi
if [ -f "$backup_version" ]; then
install -m 0644 "$backup_version" "$live_version.rollback"
mv -f "$live_version.rollback" "$live_version"
fi
supervisorctl_cmd start cliproxy >/dev/null
wait_for_proxy
}
on_exit() {
rc=$?
trap - EXIT INT TERM HUP
if [ "$rc" -ne 0 ] && [ "$swap_started" -eq 1 ]; then
rollback || true
fi
cleanup
exit "$rc"
}
trap on_exit EXIT INT TERM HUP
mkdir -p "$stage_ccs_dir"
CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest --backend plus
test -x "$stage_binary"
test -s "$stage_version"
"$stage_binary" --version >/dev/null
cp -p "$live_binary" "$backup_binary"
cp -p "$live_version" "$backup_version"
supervisorctl_cmd stop cliproxy >/dev/null
swap_started=1
mv -f "$stage_binary" "$live_binary"
mv -f "$stage_version" "$live_version"
chmod 0755 "$live_binary"
supervisorctl_cmd start cliproxy >/dev/null
wait_for_proxy
swap_started=0
CONTAINER_UPDATE
then
log 'CLIProxy Plus update failed; previous binary was restored and restarted'
exit 1
fi
installed_output="$(docker exec "$container_name" ccs cliproxy --version --backend plus 2>&1)"
log "CLIProxy Plus update completed and passed health verification: $(grep -m1 'Version:' <<<"$installed_output" | xargs)"
@@ -0,0 +1,9 @@
[Unit]
Description=Reconcile the CCS CLIProxy Docker service
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh
TimeoutStartSec=5min
@@ -0,0 +1,11 @@
[Unit]
Description=Continuously reconcile the CCS CLIProxy Docker service
[Timer]
OnBootSec=30s
OnUnitActiveSec=30s
AccuracySec=5s
Persistent=true
[Install]
WantedBy=timers.target
@@ -0,0 +1,9 @@
[Unit]
Description=Safely update CLIProxy Plus in ccs-cliproxy
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
ExecStart=/opt/cliproxy/ccs-cliproxy-safe-update.sh
TimeoutStartSec=10min
@@ -0,0 +1,10 @@
[Unit]
Description=Poll for CLIProxy Plus releases without interrupting the live binary
[Timer]
OnBootSec=3min
OnUnitActiveSec=15min
Persistent=true
[Install]
WantedBy=timers.target
-9
View File
@@ -335,7 +335,6 @@ export async function installCliproxyVersion(
const waitForPortFreeFn = deps.waitForPortFreeFn ?? waitForPortFree;
const formatInfo = deps.formatInfo ?? info;
const formatWarn = deps.formatWarn ?? warn;
const getInstalledVersion = deps.getInstalledVersion ?? getInstalledCliproxyVersion;
// Always attempt a best-effort stop first so we also catch untracked proxies
// that are running without a session lock.
@@ -352,14 +351,6 @@ export async function installCliproxyVersion(
console.log(formatWarn(`Could not stop proxy: ${result.error}`));
}
if (manager.isBinaryInstalled()) {
const label = effectiveBackend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
if (verbose)
console.log(
formatInfo(`Removing existing ${label} v${getInstalledVersion(effectiveBackend)}`)
);
manager.deleteBinary();
}
await manager.ensureBinary();
if (verbose) {
@@ -0,0 +1,151 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { getExecutableName } from '../platform-detector';
import { downloadAndInstall } from '../installer';
describe('atomic binary installation', () => {
let binPath: string;
beforeEach(() => {
binPath = fs.mkdtempSync(path.join(os.tmpdir(), 'ccs-atomic-installer-'));
});
afterEach(() => {
fs.rmSync(binPath, { recursive: true, force: true });
});
function stagingDirectories(): string[] {
return fs.readdirSync(binPath).filter((entry) => entry.startsWith('.cliproxy-install-'));
}
it('preserves the installed binary and version when verification fails', async () => {
const binaryPath = path.join(binPath, getExecutableName('original'));
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await expect(
downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: false,
expected: 'expected-checksum',
actual: 'actual-checksum',
}),
extractArchiveFn: async () => {
throw new Error('extract should not run');
},
}
)
).rejects.toThrow('Checksum mismatch');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
expect(stagingDirectories()).toEqual([]);
});
it('atomically swaps the verified staged binary and then records its version', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
}
);
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('new-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.7.1');
expect(stagingDirectories()).toEqual([]);
});
it('preserves the installed binary and version when the atomic swap fails', async () => {
const binaryName = getExecutableName('original');
const binaryPath = path.join(binPath, binaryName);
const versionPath = path.join(binPath, '.version');
fs.writeFileSync(binaryPath, 'old-binary');
fs.writeFileSync(versionPath, '6.6.80');
await expect(
downloadAndInstall(
{
version: '6.7.1',
releaseUrl: 'https://example.invalid',
binPath,
maxRetries: 1,
verbose: false,
forceVersion: true,
skipAutoUpdate: false,
allowInstall: true,
backend: 'original',
},
false,
{
downloadWithRetryFn: async (_url, archivePath) => {
fs.writeFileSync(archivePath, 'downloaded-archive');
return { success: true, filePath: archivePath, retries: 0 };
},
verifyChecksumFn: async () => ({
valid: true,
expected: 'checksum',
actual: 'checksum',
}),
extractArchiveFn: async (_archivePath, destination) => {
fs.writeFileSync(path.join(destination, binaryName), 'new-binary');
},
renameSyncFn: () => {
throw Object.assign(new Error('replacement blocked'), { code: 'EPERM' });
},
}
)
).rejects.toThrow('replacement blocked');
expect(fs.readFileSync(binaryPath, 'utf8')).toBe('old-binary');
expect(fs.readFileSync(versionPath, 'utf8')).toBe('6.6.80');
expect(stagingDirectories()).toEqual([]);
});
});
@@ -176,6 +176,38 @@ describe('installCliproxyVersion', () => {
expect(calls.ensureBinary).toBe(1);
});
it('keeps the installed binary in place while the replacement is prepared', async () => {
const calls = {
deleteBinary: 0,
ensureBinary: 0,
};
const binaryManager = await import(
`../../binary-manager?binary-manager-atomic-install=${Date.now()}`
);
await binaryManager.installCliproxyVersion('6.7.1', false, 'plus', {
createManager: () => ({
isBinaryInstalled: () => true,
deleteBinary: () => {
calls.deleteBinary += 1;
},
ensureBinary: async () => {
calls.ensureBinary += 1;
return '/tmp/ccs-bin/plus/cli-proxy-api-plus';
},
}),
stopProxyFn: async () => ({ stopped: false, error: 'No active CLIProxy session found' }),
waitForPortFreeFn: async () => true,
formatInfo: (message: string) => message,
formatWarn: (message: string) => message,
getInstalledVersion: () => '6.6.80',
});
expect(calls.deleteBinary).toBe(0);
expect(calls.ensureBinary).toBe(1);
});
it('waits for the port that was actually stopped before continuing install', async () => {
let waitedPort: number | undefined;
+43 -43
View File
@@ -19,63 +19,53 @@ import { extractArchive } from './extractor';
import { writeInstalledVersion } from './version-cache';
import { ProgressIndicator } from '../../utils/progress-indicator';
import { ok } from '../../utils/ui';
import { BinaryError, NetworkError } from '../../errors/error-types';
interface DownloadAndInstallDeps {
downloadWithRetryFn?: typeof downloadWithRetry;
verifyChecksumFn?: typeof verifyChecksum;
extractArchiveFn?: typeof extractArchive;
renameSyncFn?: typeof fs.renameSync;
}
/**
* Download and install the binary
*/
export async function downloadAndInstall(
config: BinaryManagerConfig,
verbose = false
verbose = false,
deps: DownloadAndInstallDeps = {}
): Promise<void> {
const backend = config.backend ?? DEFAULT_BACKEND;
const platform = detectPlatform(config.version, backend);
const downloadUrl = getDownloadUrl(config.version, backend);
const checksumsUrl = getChecksumsUrl(config.version, backend);
const backendLabel = backend === 'plus' ? 'CLIProxy Plus' : 'CLIProxy';
const downloadWithRetryFn = deps.downloadWithRetryFn ?? downloadWithRetry;
const verifyChecksumFn = deps.verifyChecksumFn ?? verifyChecksum;
const extractArchiveFn = deps.extractArchiveFn ?? extractArchive;
const renameSyncFn = deps.renameSyncFn ?? fs.renameSync;
fs.mkdirSync(config.binPath, { recursive: true });
// Delete existing binary before install to prevent mismatched binaries.
// Abort if binary is currently running (ETXTBSY) — cannot replace in-use binary.
// Happens in Docker when dashboard tries to update while bootstrap's instance is active.
const existingBinary = path.join(config.binPath, getExecutableName(backend));
if (fs.existsSync(existingBinary)) {
try {
fs.unlinkSync(existingBinary);
if (verbose) console.error(`[cliproxy] Removed existing binary: ${existingBinary}`);
} catch (error: unknown) {
const code =
error instanceof Error && 'code' in error ? (error as { code: string }).code : '';
// ETXTBSY: Linux-specific error when unlinking a running executable.
// EBUSY on Windows may mean something different (mount point, etc.),
// so only treat ETXTBSY as "binary in use" to avoid misleading messages.
if (code === 'ETXTBSY') {
if (verbose)
console.error(`[cliproxy] Binary is running, cannot replace: ${existingBinary}`);
throw new Error(
'CLIProxy binary is currently running and cannot be replaced. Restart the container to apply the update.'
);
}
throw error;
}
}
const archivePath = path.join(config.binPath, `cliproxy-archive.${platform.extension}`);
const stagingPath = fs.mkdtempSync(path.join(config.binPath, '.cliproxy-install-'));
const archivePath = path.join(stagingPath, `cliproxy-archive.${platform.extension}`);
const stagedBinary = path.join(stagingPath, getExecutableName(backend));
const installedBinary = path.join(config.binPath, getExecutableName(backend));
const spinner = new ProgressIndicator(`Downloading ${backendLabel} v${config.version}`);
spinner.start();
try {
const result = await downloadWithRetry(downloadUrl, archivePath, {
const result = await downloadWithRetryFn(downloadUrl, archivePath, {
maxRetries: config.maxRetries,
verbose,
});
if (!result.success) {
spinner.fail('Download failed');
throw new Error(result.error || 'Download failed after retries');
throw new NetworkError(result.error || 'Download failed after retries', downloadUrl);
}
spinner.update('Verifying checksum');
const checksumResult = await verifyChecksum(
const checksumResult = await verifyChecksumFn(
archivePath,
platform.binaryName,
checksumsUrl,
@@ -84,29 +74,36 @@ export async function downloadAndInstall(
if (!checksumResult.valid) {
spinner.fail('Checksum mismatch');
fs.unlinkSync(archivePath);
throw new Error(
throw new BinaryError(
`Checksum mismatch for ${platform.binaryName}\nExpected: ${checksumResult.expected}\n` +
`Actual: ${checksumResult.actual}\n\nManual download: ${downloadUrl}`
`Actual: ${checksumResult.actual}\n\nManual download: ${downloadUrl}`,
stagedBinary
);
}
spinner.update('Extracting binary');
await extractArchive(archivePath, config.binPath, platform.extension, verbose, backend);
spinner.succeed(`${backendLabel} ready`);
fs.unlinkSync(archivePath);
const binaryPath = path.join(config.binPath, getExecutableName(backend));
if (platform.os !== 'windows' && fs.existsSync(binaryPath)) {
fs.chmodSync(binaryPath, 0o755);
if (verbose) console.error(`[cliproxy] Set executable permissions: ${binaryPath}`);
await extractArchiveFn(archivePath, stagingPath, platform.extension, verbose, backend);
if (!fs.existsSync(stagedBinary) || !fs.statSync(stagedBinary).isFile()) {
throw new BinaryError(
`Extracted archive did not contain ${getExecutableName(backend)}`,
stagedBinary
);
}
if (platform.os !== 'windows') {
fs.chmodSync(stagedBinary, 0o755);
if (verbose) console.error(`[cliproxy] Set executable permissions: ${stagedBinary}`);
}
renameSyncFn(stagedBinary, installedBinary);
writeInstalledVersion(config.binPath, config.version);
spinner.succeed(`${backendLabel} ready`);
console.log(ok(`${backendLabel} v${config.version} installed successfully`));
} catch (error) {
spinner.fail('Installation failed');
throw error;
} finally {
fs.rmSync(stagingPath, { recursive: true, force: true });
}
}
@@ -124,7 +121,10 @@ export function deleteBinary(binPath: string, verbose = false, backend?: CLIProx
const code =
error instanceof Error && 'code' in error ? (error as { code: string }).code : '';
if (code === 'ETXTBSY') {
throw new Error('CLIProxy binary is currently running and cannot be deleted.');
throw new BinaryError(
'CLIProxy binary is currently running and cannot be deleted.',
binaryPath
);
}
throw error;
}
+2 -2
View File
@@ -11,7 +11,7 @@ import {
isNewerVersion,
isVersionFaulty,
} from './version-checker';
import { downloadAndInstall, deleteBinary, getBinaryPath } from './installer';
import { downloadAndInstall, getBinaryPath } from './installer';
import { info, warn } from '../../utils/ui';
import { isCliproxyRunning } from '../services/stats-fetcher';
import { resolveLifecyclePort } from '../config/port-manager';
@@ -93,7 +93,7 @@ async function handleAutoUpdate(config: BinaryManagerConfig, verbose: boolean):
} else {
console.log(info(updateMsg));
console.log(info(`Updating ${backendLabel}...`));
deleteBinary(config.binPath, verbose, backend);
// The installer stages and validates the replacement before atomically swapping it.
config.version = targetVersion;
await downloadAndInstall(config, verbose);
}
@@ -0,0 +1,50 @@
import { readFileSync } from 'fs';
import { describe, expect, it } from 'bun:test';
const updateScript = readFileSync('docker/host/ccs-cliproxy-safe-update.sh', 'utf8');
const reconcileScript = readFileSync('docker/host/ccs-cliproxy-reconcile.sh', 'utf8');
const updateService = readFileSync('docker/host/systemd/ccs-cliproxy-update.service', 'utf8');
const updateTimer = readFileSync('docker/host/systemd/ccs-cliproxy-update.timer', 'utf8');
const reconcileService = readFileSync('docker/host/systemd/ccs-cliproxy-reconcile.service', 'utf8');
const reconcileTimer = readFileSync('docker/host/systemd/ccs-cliproxy-reconcile.timer', 'utf8');
describe('CLIProxy Docker host continuity assets', () => {
it('stages and validates the replacement before stopping the live proxy', () => {
const stageIndex = updateScript.indexOf('CCS_DIR="$stage_ccs_dir" ccs cliproxy --latest');
const validateIndex = updateScript.indexOf('"$stage_binary" --version');
const stopIndex = updateScript.indexOf('supervisorctl_cmd stop cliproxy', validateIndex);
expect(stageIndex).toBeGreaterThan(-1);
expect(validateIndex).toBeGreaterThan(stageIndex);
expect(stopIndex).toBeGreaterThan(validateIndex);
});
it('serializes updates and reconciliation with the same lock', () => {
expect(updateScript).toContain('/run/lock/ccs-cliproxy-maintenance.lock');
expect(reconcileScript).toContain('/run/lock/ccs-cliproxy-maintenance.lock');
expect(updateScript).toContain('flock -n 9');
expect(reconcileScript).toContain('flock -n 9');
});
it('rolls back failed swaps and health-checks both services', () => {
expect(updateScript).toContain('rollback()');
expect(updateScript).toContain('previous-binary');
expect(updateScript).toContain('wait_for_proxy');
expect(reconcileScript).toContain('http://127.0.0.1:3000/');
expect(reconcileScript).toContain('http://127.0.0.1:8317/');
});
it('recreates missing containers and escalates unhealthy recovery', () => {
expect(reconcileScript).toContain('docker compose -f "$compose_file" up -d --no-build');
expect(reconcileScript).toContain('restart ccs-dashboard cliproxy');
expect(reconcileScript).toContain('docker restart "$container_name"');
});
it('installs executable services on bounded timers', () => {
expect(updateService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-safe-update.sh');
expect(updateService).toContain('TimeoutStartSec=10min');
expect(updateTimer).toContain('OnUnitActiveSec=15min');
expect(reconcileService).toContain('ExecStart=/opt/cliproxy/ccs-cliproxy-reconcile.sh');
expect(reconcileTimer).toContain('OnUnitActiveSec=30s');
});
});