fix(proxy): preserve running daemons on restart failure

This commit is contained in:
Wooseong Kim
2026-04-20 14:51:32 +09:00
parent dfd070c9c6
commit db32d15d86
4 changed files with 233 additions and 56 deletions
+39 -13
View File
@@ -133,29 +133,55 @@ async function handleStatus(args: string[] = []): Promise<number> {
return showHelp(); return showHelp();
} }
const profileName = findPositionalArg(args); const profileName = findPositionalArg(args);
const running = profileName const printStatus = (status: Awaited<ReturnType<typeof getOpenAICompatProxyStatus>>) => {
? [await getOpenAICompatProxyStatus(profileName)].filter((status) => status.running) console.log(
: (await listOpenAICompatProxyStatuses()).filter((status) => status.running); status.running
if (running.length === 0) { ? ok(`Proxy running on port ${status.port}`)
console.log(info('Proxy is not running')); : info(`Proxy is not running${status.port ? ` (last known port ${status.port})` : ''}`)
return 0; );
} if (status.host && status.port) {
for (const status of running) {
console.log(ok(`Proxy running on port ${status.port}`));
if (status.host) {
console.log(` Host: ${status.host}`); console.log(` Host: ${status.host}`);
console.log(` Local URL: http://${status.host}:${status.port}`); console.log(` Local URL: http://${status.host}:${status.port}`);
} }
console.log(` Profile: ${status.profileName}`); if (status.profileName) {
console.log(` Base URL: ${status.baseUrl}`); console.log(` Profile: ${status.profileName}`);
}
if (status.baseUrl) {
console.log(` Base URL: ${status.baseUrl}`);
}
if (status.model) { if (status.model) {
console.log(` Model: ${status.model}`); console.log(` Model: ${status.model}`);
} }
if (status.pid) { if (status.pid) {
console.log(` PID: ${status.pid}`); console.log(` PID: ${status.pid}`);
} }
};
if (profileName) {
const status = await getOpenAICompatProxyStatus(profileName);
if (!status.running && !status.profileName) {
console.log(info('Proxy is not running'));
return 0;
}
printStatus(status);
return 0;
} }
const status = await getOpenAICompatProxyStatus();
if (!status.running && !status.profileName) {
console.log(info('Proxy is not running'));
return 0;
}
if (status.running && !status.profileName) {
const running = (await listOpenAICompatProxyStatuses()).filter((entry) => entry.running);
for (const entry of running) {
printStatus(entry);
}
return 0;
}
printStatus(status);
return 0; return 0;
} }
+117 -43
View File
@@ -42,6 +42,11 @@ export interface StartOpenAICompatProxyResult {
error?: string; error?: string;
} }
interface OpenAICompatProxyLaunchResult extends StartOpenAICompatProxyResult {
commitState?: () => void;
stop?: () => Promise<void>;
}
function generateProxyAuthToken(): string { function generateProxyAuthToken(): string {
return crypto.randomBytes(24).toString('hex'); return crypto.randomBytes(24).toString('hex');
} }
@@ -93,6 +98,30 @@ async function isPortOccupied(port: number): Promise<boolean> {
}); });
} }
async function terminateDaemonProcess(pid?: number): Promise<void> {
if (!pid) {
return;
}
try {
process.kill(pid, 'SIGTERM');
let attempts = 0;
while (attempts < 10) {
await new Promise((resolve) => setTimeout(resolve, 200));
try {
process.kill(pid, 0);
attempts += 1;
} catch {
return;
}
}
process.kill(pid, 'SIGKILL');
} catch {
// Best-effort cleanup for a daemon we just spawned.
}
}
async function findOpenAICompatProxyPort( async function findOpenAICompatProxyPort(
excludedPorts: ReadonlySet<number> = new Set() excludedPorts: ReadonlySet<number> = new Set()
): Promise<number> { ): Promise<number> {
@@ -397,17 +426,35 @@ export async function startOpenAICompatProxy(
}; };
} }
const startOnPort = (port: number): Promise<StartOpenAICompatProxyResult> => const launchOnPort = (
port: number,
persistState: boolean
): Promise<OpenAICompatProxyLaunchResult> =>
new Promise((resolve) => { new Promise((resolve) => {
let resolved = false; let resolved = false;
let timeout: NodeJS.Timeout | null = null; let timeout: NodeJS.Timeout | null = null;
const authToken = generateProxyAuthToken(); const authToken = generateProxyAuthToken();
const commitState = () => {
if (proc.pid) {
writeOpenAICompatProxyPid(profile.profileName, proc.pid);
}
writeOpenAICompatProxySession({
profileName: profile.profileName,
settingsPath: profile.settingsPath,
host,
port,
baseUrl: profile.baseUrl,
authToken,
model: profile.model,
insecure: options.insecure,
});
};
const finish = (result: StartOpenAICompatProxyResult) => { const finish = (result: OpenAICompatProxyLaunchResult) => {
if (resolved) return; if (resolved) return;
resolved = true; resolved = true;
if (timeout) clearTimeout(timeout); if (timeout) clearTimeout(timeout);
if (!result.success) { if (!result.success && persistState) {
removeOpenAICompatProxyPid(profile.profileName); removeOpenAICompatProxyPid(profile.profileName);
removeOpenAICompatProxySession(profile.profileName); removeOpenAICompatProxySession(profile.profileName);
} }
@@ -435,25 +482,28 @@ export async function startOpenAICompatProxy(
); );
proc.unref(); proc.unref();
if (proc.pid) { if (persistState) {
writeOpenAICompatProxyPid(profile.profileName, proc.pid); commitState();
} }
writeOpenAICompatProxySession({
profileName: profile.profileName,
settingsPath: profile.settingsPath,
host,
port,
baseUrl: profile.baseUrl,
authToken,
model: profile.model,
insecure: options.insecure,
});
let attempts = 0; let attempts = 0;
const poll = async () => { const poll = async () => {
attempts += 1; attempts += 1;
if (await isOpenAICompatProxyRunning(port)) { if (await isOpenAICompatProxyRunning(port)) {
finish({ success: true, pid: proc.pid, port, authToken }); finish({
success: true,
pid: proc.pid,
port,
authToken,
...(persistState
? {}
: {
commitState,
stop: async () => {
await terminateDaemonProcess(proc.pid);
},
}),
});
return; return;
} }
if (attempts >= 30) { if (attempts >= 30) {
@@ -496,40 +546,64 @@ export async function startOpenAICompatProxy(
}); });
}); });
if (requiresExactPort) { const launchProxy = async (persistState: boolean): Promise<OpenAICompatProxyLaunchResult> => {
return startOnPort(preferredPort); if (requiresExactPort) {
} return launchOnPort(preferredPort, persistState);
}
const attemptedPorts = new Set<number>(); const attemptedPorts = new Set<number>();
let lastResult: StartOpenAICompatProxyResult | null = null; let lastResult: OpenAICompatProxyLaunchResult | null = null;
for (let attempt = 0; attempt < 3; attempt += 1) { for (let attempt = 0; attempt < 3; attempt += 1) {
const port = await findOpenAICompatProxyPortNear(preferredPort, attemptedPorts); const port = await findOpenAICompatProxyPortNear(preferredPort, attemptedPorts);
if (port === 0) { if (port === 0) {
return { return {
success: false,
port: preferredPort,
error: `No free proxy port found in range ${OPENAI_COMPAT_PROXY_DEFAULT_PORT}-${OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10}`,
};
}
const result = await launchOnPort(port, persistState);
if (result.success) {
return result;
}
lastResult = result;
attemptedPorts.add(port);
if (!(await isPortOccupied(port))) {
return result;
}
}
return (
lastResult ?? {
success: false, success: false,
port: preferredPort, port: preferredPort,
error: `No free proxy port found in range ${OPENAI_COMPAT_PROXY_DEFAULT_PORT}-${OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10}`, error: 'Failed to start proxy',
}
);
};
if (status.running) {
const launched = await launchProxy(false);
if (!launched.success) {
return launched;
}
const stopped = await stopOpenAICompatProxyUnlocked(profile.profileName);
if (!stopped.success) {
await launched.stop?.();
return {
success: false,
port: launched.port,
error: stopped.error || 'Failed to replace the running proxy',
}; };
} }
const result = await startOnPort(port); launched.commitState?.();
if (result.success) { return launched;
return result;
}
lastResult = result;
attemptedPorts.add(port);
if (!(await isPortOccupied(port))) {
return result;
}
} }
return ( return launchProxy(true);
lastResult ?? {
success: false,
port: preferredPort,
error: 'Failed to start proxy',
}
);
}); });
} }
+28
View File
@@ -52,6 +52,34 @@ describe('proxy command e2e', () => {
expect(help.stdout).toContain('stop [profile] Stop the running proxy (or all proxies when omitted)'); expect(help.stdout).toContain('stop [profile] Stop the running proxy (or all proxies when omitted)');
}); });
it('shows the last-known proxy state when no proxy is currently running', async () => {
const stalePort = await getPort();
const proxyDir = path.join(tempDir, '.ccs', 'proxy');
fs.mkdirSync(proxyDir, { recursive: true });
fs.writeFileSync(
path.join(proxyDir, 'stale.session.json'),
JSON.stringify(
{
profileName: 'stale',
settingsPath: path.join(tempDir, '.ccs', 'stale.settings.json'),
host: '127.0.0.1',
port: stalePort,
baseUrl: 'http://127.0.0.1:11434',
authToken: 'deadbeef',
model: 'qwen3-coder',
},
null,
2
) + '\n',
'utf8'
);
const status = runCli(['proxy', 'status']);
expect(status.status).toBe(0);
expect(status.stdout).toContain(`Proxy is not running (last known port ${stalePort})`);
expect(status.stdout).toContain('Profile: stale');
});
it('starts, reports status, activates, and stops via the built CLI', async () => { it('starts, reports status, activates, and stops via the built CLI', async () => {
const port = await getPort(); const port = await getPort();
const ccsDir = path.join(tempDir, '.ccs'); const ccsDir = path.join(tempDir, '.ccs');
@@ -303,4 +303,53 @@ describe('openai proxy daemon lifecycle', () => {
server.stop(true); server.stop(true);
} }
}); });
it('keeps the existing proxy running if replacement startup fails', async () => {
const firstPort = await getPort();
const occupiedPort = await getPort();
const busyServer = Bun.serve({
port: occupiedPort,
hostname: '127.0.0.1',
fetch: () => new Response('busy'),
});
try {
const settingsPath = path.join(tempDir, 'rollback.settings.json');
fs.writeFileSync(
settingsPath,
JSON.stringify({
env: {
ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434',
ANTHROPIC_AUTH_TOKEN: 'ollama-rollback',
ANTHROPIC_MODEL: 'qwen3-coder',
CCS_DROID_PROVIDER: 'generic-chat-completion-api',
},
}),
'utf8'
);
const profile = resolveOpenAICompatProfileConfig('rollback', settingsPath, {
ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434',
ANTHROPIC_AUTH_TOKEN: 'ollama-rollback',
ANTHROPIC_MODEL: 'qwen3-coder',
CCS_DROID_PROVIDER: 'generic-chat-completion-api',
});
if (!profile) {
throw new Error('Expected a rollback OpenAI-compatible profile');
}
const firstStart = await startOpenAICompatProxy(profile, { port: firstPort });
expect(firstStart.success).toBe(true);
const restarted = await startOpenAICompatProxy(profile, { port: occupiedPort });
expect(restarted.success).toBe(false);
const status = await getOpenAICompatProxyStatus('rollback');
expect(status.running).toBe(true);
expect(status.port).toBe(firstPort);
expect((await fetch(`http://127.0.0.1:${firstPort}/health`)).status).toBe(200);
} finally {
busyServer.stop(true);
}
});
}); });