mirror of
https://github.com/tiennm99/ccs.git
synced 2026-09-02 16:19:27 +00:00
fix(proxy): preserve running daemons on restart failure
This commit is contained in:
@@ -133,29 +133,55 @@ async function handleStatus(args: string[] = []): Promise<number> {
|
|||||||
return showHelp();
|
return showHelp();
|
||||||
}
|
}
|
||||||
const profileName = findPositionalArg(args);
|
const profileName = findPositionalArg(args);
|
||||||
const running = profileName
|
const printStatus = (status: Awaited<ReturnType<typeof getOpenAICompatProxyStatus>>) => {
|
||||||
? [await getOpenAICompatProxyStatus(profileName)].filter((status) => status.running)
|
console.log(
|
||||||
: (await listOpenAICompatProxyStatuses()).filter((status) => status.running);
|
status.running
|
||||||
if (running.length === 0) {
|
? ok(`Proxy running on port ${status.port}`)
|
||||||
console.log(info('Proxy is not running'));
|
: info(`Proxy is not running${status.port ? ` (last known port ${status.port})` : ''}`)
|
||||||
return 0;
|
);
|
||||||
}
|
if (status.host && status.port) {
|
||||||
|
|
||||||
for (const status of running) {
|
|
||||||
console.log(ok(`Proxy running on port ${status.port}`));
|
|
||||||
if (status.host) {
|
|
||||||
console.log(` Host: ${status.host}`);
|
console.log(` Host: ${status.host}`);
|
||||||
console.log(` Local URL: http://${status.host}:${status.port}`);
|
console.log(` Local URL: http://${status.host}:${status.port}`);
|
||||||
}
|
}
|
||||||
console.log(` Profile: ${status.profileName}`);
|
if (status.profileName) {
|
||||||
console.log(` Base URL: ${status.baseUrl}`);
|
console.log(` Profile: ${status.profileName}`);
|
||||||
|
}
|
||||||
|
if (status.baseUrl) {
|
||||||
|
console.log(` Base URL: ${status.baseUrl}`);
|
||||||
|
}
|
||||||
if (status.model) {
|
if (status.model) {
|
||||||
console.log(` Model: ${status.model}`);
|
console.log(` Model: ${status.model}`);
|
||||||
}
|
}
|
||||||
if (status.pid) {
|
if (status.pid) {
|
||||||
console.log(` PID: ${status.pid}`);
|
console.log(` PID: ${status.pid}`);
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (profileName) {
|
||||||
|
const status = await getOpenAICompatProxyStatus(profileName);
|
||||||
|
if (!status.running && !status.profileName) {
|
||||||
|
console.log(info('Proxy is not running'));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
printStatus(status);
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const status = await getOpenAICompatProxyStatus();
|
||||||
|
if (!status.running && !status.profileName) {
|
||||||
|
console.log(info('Proxy is not running'));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status.running && !status.profileName) {
|
||||||
|
const running = (await listOpenAICompatProxyStatuses()).filter((entry) => entry.running);
|
||||||
|
for (const entry of running) {
|
||||||
|
printStatus(entry);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
printStatus(status);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+117
-43
@@ -42,6 +42,11 @@ export interface StartOpenAICompatProxyResult {
|
|||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface OpenAICompatProxyLaunchResult extends StartOpenAICompatProxyResult {
|
||||||
|
commitState?: () => void;
|
||||||
|
stop?: () => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
function generateProxyAuthToken(): string {
|
function generateProxyAuthToken(): string {
|
||||||
return crypto.randomBytes(24).toString('hex');
|
return crypto.randomBytes(24).toString('hex');
|
||||||
}
|
}
|
||||||
@@ -93,6 +98,30 @@ async function isPortOccupied(port: number): Promise<boolean> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function terminateDaemonProcess(pid?: number): Promise<void> {
|
||||||
|
if (!pid) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
process.kill(pid, 'SIGTERM');
|
||||||
|
let attempts = 0;
|
||||||
|
while (attempts < 10) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
attempts += 1;
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
process.kill(pid, 'SIGKILL');
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup for a daemon we just spawned.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function findOpenAICompatProxyPort(
|
async function findOpenAICompatProxyPort(
|
||||||
excludedPorts: ReadonlySet<number> = new Set()
|
excludedPorts: ReadonlySet<number> = new Set()
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
@@ -397,17 +426,35 @@ export async function startOpenAICompatProxy(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const startOnPort = (port: number): Promise<StartOpenAICompatProxyResult> =>
|
const launchOnPort = (
|
||||||
|
port: number,
|
||||||
|
persistState: boolean
|
||||||
|
): Promise<OpenAICompatProxyLaunchResult> =>
|
||||||
new Promise((resolve) => {
|
new Promise((resolve) => {
|
||||||
let resolved = false;
|
let resolved = false;
|
||||||
let timeout: NodeJS.Timeout | null = null;
|
let timeout: NodeJS.Timeout | null = null;
|
||||||
const authToken = generateProxyAuthToken();
|
const authToken = generateProxyAuthToken();
|
||||||
|
const commitState = () => {
|
||||||
|
if (proc.pid) {
|
||||||
|
writeOpenAICompatProxyPid(profile.profileName, proc.pid);
|
||||||
|
}
|
||||||
|
writeOpenAICompatProxySession({
|
||||||
|
profileName: profile.profileName,
|
||||||
|
settingsPath: profile.settingsPath,
|
||||||
|
host,
|
||||||
|
port,
|
||||||
|
baseUrl: profile.baseUrl,
|
||||||
|
authToken,
|
||||||
|
model: profile.model,
|
||||||
|
insecure: options.insecure,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const finish = (result: StartOpenAICompatProxyResult) => {
|
const finish = (result: OpenAICompatProxyLaunchResult) => {
|
||||||
if (resolved) return;
|
if (resolved) return;
|
||||||
resolved = true;
|
resolved = true;
|
||||||
if (timeout) clearTimeout(timeout);
|
if (timeout) clearTimeout(timeout);
|
||||||
if (!result.success) {
|
if (!result.success && persistState) {
|
||||||
removeOpenAICompatProxyPid(profile.profileName);
|
removeOpenAICompatProxyPid(profile.profileName);
|
||||||
removeOpenAICompatProxySession(profile.profileName);
|
removeOpenAICompatProxySession(profile.profileName);
|
||||||
}
|
}
|
||||||
@@ -435,25 +482,28 @@ export async function startOpenAICompatProxy(
|
|||||||
);
|
);
|
||||||
|
|
||||||
proc.unref();
|
proc.unref();
|
||||||
if (proc.pid) {
|
if (persistState) {
|
||||||
writeOpenAICompatProxyPid(profile.profileName, proc.pid);
|
commitState();
|
||||||
}
|
}
|
||||||
writeOpenAICompatProxySession({
|
|
||||||
profileName: profile.profileName,
|
|
||||||
settingsPath: profile.settingsPath,
|
|
||||||
host,
|
|
||||||
port,
|
|
||||||
baseUrl: profile.baseUrl,
|
|
||||||
authToken,
|
|
||||||
model: profile.model,
|
|
||||||
insecure: options.insecure,
|
|
||||||
});
|
|
||||||
|
|
||||||
let attempts = 0;
|
let attempts = 0;
|
||||||
const poll = async () => {
|
const poll = async () => {
|
||||||
attempts += 1;
|
attempts += 1;
|
||||||
if (await isOpenAICompatProxyRunning(port)) {
|
if (await isOpenAICompatProxyRunning(port)) {
|
||||||
finish({ success: true, pid: proc.pid, port, authToken });
|
finish({
|
||||||
|
success: true,
|
||||||
|
pid: proc.pid,
|
||||||
|
port,
|
||||||
|
authToken,
|
||||||
|
...(persistState
|
||||||
|
? {}
|
||||||
|
: {
|
||||||
|
commitState,
|
||||||
|
stop: async () => {
|
||||||
|
await terminateDaemonProcess(proc.pid);
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (attempts >= 30) {
|
if (attempts >= 30) {
|
||||||
@@ -496,40 +546,64 @@ export async function startOpenAICompatProxy(
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
if (requiresExactPort) {
|
const launchProxy = async (persistState: boolean): Promise<OpenAICompatProxyLaunchResult> => {
|
||||||
return startOnPort(preferredPort);
|
if (requiresExactPort) {
|
||||||
}
|
return launchOnPort(preferredPort, persistState);
|
||||||
|
}
|
||||||
|
|
||||||
const attemptedPorts = new Set<number>();
|
const attemptedPorts = new Set<number>();
|
||||||
let lastResult: StartOpenAICompatProxyResult | null = null;
|
let lastResult: OpenAICompatProxyLaunchResult | null = null;
|
||||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||||
const port = await findOpenAICompatProxyPortNear(preferredPort, attemptedPorts);
|
const port = await findOpenAICompatProxyPortNear(preferredPort, attemptedPorts);
|
||||||
if (port === 0) {
|
if (port === 0) {
|
||||||
return {
|
return {
|
||||||
|
success: false,
|
||||||
|
port: preferredPort,
|
||||||
|
error: `No free proxy port found in range ${OPENAI_COMPAT_PROXY_DEFAULT_PORT}-${OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await launchOnPort(port, persistState);
|
||||||
|
if (result.success) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
lastResult = result;
|
||||||
|
attemptedPorts.add(port);
|
||||||
|
if (!(await isPortOccupied(port))) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
lastResult ?? {
|
||||||
success: false,
|
success: false,
|
||||||
port: preferredPort,
|
port: preferredPort,
|
||||||
error: `No free proxy port found in range ${OPENAI_COMPAT_PROXY_DEFAULT_PORT}-${OPENAI_COMPAT_PROXY_DEFAULT_PORT + 10}`,
|
error: 'Failed to start proxy',
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (status.running) {
|
||||||
|
const launched = await launchProxy(false);
|
||||||
|
if (!launched.success) {
|
||||||
|
return launched;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stopped = await stopOpenAICompatProxyUnlocked(profile.profileName);
|
||||||
|
if (!stopped.success) {
|
||||||
|
await launched.stop?.();
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
port: launched.port,
|
||||||
|
error: stopped.error || 'Failed to replace the running proxy',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await startOnPort(port);
|
launched.commitState?.();
|
||||||
if (result.success) {
|
return launched;
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
lastResult = result;
|
|
||||||
attemptedPorts.add(port);
|
|
||||||
if (!(await isPortOccupied(port))) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return launchProxy(true);
|
||||||
lastResult ?? {
|
|
||||||
success: false,
|
|
||||||
port: preferredPort,
|
|
||||||
error: 'Failed to start proxy',
|
|
||||||
}
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,6 +52,34 @@ describe('proxy command e2e', () => {
|
|||||||
expect(help.stdout).toContain('stop [profile] Stop the running proxy (or all proxies when omitted)');
|
expect(help.stdout).toContain('stop [profile] Stop the running proxy (or all proxies when omitted)');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shows the last-known proxy state when no proxy is currently running', async () => {
|
||||||
|
const stalePort = await getPort();
|
||||||
|
const proxyDir = path.join(tempDir, '.ccs', 'proxy');
|
||||||
|
fs.mkdirSync(proxyDir, { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(proxyDir, 'stale.session.json'),
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
profileName: 'stale',
|
||||||
|
settingsPath: path.join(tempDir, '.ccs', 'stale.settings.json'),
|
||||||
|
host: '127.0.0.1',
|
||||||
|
port: stalePort,
|
||||||
|
baseUrl: 'http://127.0.0.1:11434',
|
||||||
|
authToken: 'deadbeef',
|
||||||
|
model: 'qwen3-coder',
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
) + '\n',
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const status = runCli(['proxy', 'status']);
|
||||||
|
expect(status.status).toBe(0);
|
||||||
|
expect(status.stdout).toContain(`Proxy is not running (last known port ${stalePort})`);
|
||||||
|
expect(status.stdout).toContain('Profile: stale');
|
||||||
|
});
|
||||||
|
|
||||||
it('starts, reports status, activates, and stops via the built CLI', async () => {
|
it('starts, reports status, activates, and stops via the built CLI', async () => {
|
||||||
const port = await getPort();
|
const port = await getPort();
|
||||||
const ccsDir = path.join(tempDir, '.ccs');
|
const ccsDir = path.join(tempDir, '.ccs');
|
||||||
|
|||||||
@@ -303,4 +303,53 @@ describe('openai proxy daemon lifecycle', () => {
|
|||||||
server.stop(true);
|
server.stop(true);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('keeps the existing proxy running if replacement startup fails', async () => {
|
||||||
|
const firstPort = await getPort();
|
||||||
|
const occupiedPort = await getPort();
|
||||||
|
const busyServer = Bun.serve({
|
||||||
|
port: occupiedPort,
|
||||||
|
hostname: '127.0.0.1',
|
||||||
|
fetch: () => new Response('busy'),
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const settingsPath = path.join(tempDir, 'rollback.settings.json');
|
||||||
|
fs.writeFileSync(
|
||||||
|
settingsPath,
|
||||||
|
JSON.stringify({
|
||||||
|
env: {
|
||||||
|
ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434',
|
||||||
|
ANTHROPIC_AUTH_TOKEN: 'ollama-rollback',
|
||||||
|
ANTHROPIC_MODEL: 'qwen3-coder',
|
||||||
|
CCS_DROID_PROVIDER: 'generic-chat-completion-api',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
|
||||||
|
const profile = resolveOpenAICompatProfileConfig('rollback', settingsPath, {
|
||||||
|
ANTHROPIC_BASE_URL: 'http://127.0.0.1:11434',
|
||||||
|
ANTHROPIC_AUTH_TOKEN: 'ollama-rollback',
|
||||||
|
ANTHROPIC_MODEL: 'qwen3-coder',
|
||||||
|
CCS_DROID_PROVIDER: 'generic-chat-completion-api',
|
||||||
|
});
|
||||||
|
if (!profile) {
|
||||||
|
throw new Error('Expected a rollback OpenAI-compatible profile');
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstStart = await startOpenAICompatProxy(profile, { port: firstPort });
|
||||||
|
expect(firstStart.success).toBe(true);
|
||||||
|
|
||||||
|
const restarted = await startOpenAICompatProxy(profile, { port: occupiedPort });
|
||||||
|
expect(restarted.success).toBe(false);
|
||||||
|
|
||||||
|
const status = await getOpenAICompatProxyStatus('rollback');
|
||||||
|
expect(status.running).toBe(true);
|
||||||
|
expect(status.port).toBe(firstPort);
|
||||||
|
expect((await fetch(`http://127.0.0.1:${firstPort}/health`)).status).toBe(200);
|
||||||
|
} finally {
|
||||||
|
busyServer.stop(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user