mirror of
https://github.com/tiennm99/coolify.git
synced 2026-04-17 17:21:04 +00:00
Fix complex status logic: handle degraded sub-resources and mixed running+starting states
- Add support for degraded status from sub-resources as highest priority - Handle mixed running+starting state to show service as not fully ready - Update state priority hierarchy from 8 to 10 levels - Add comprehensive test coverage for new status scenarios 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -16,14 +16,16 @@ use Illuminate\Support\Facades\Log;
|
|||||||
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
|
* UI components transform this to human-readable format (e.g., "Running (Healthy)").
|
||||||
*
|
*
|
||||||
* State Priority (highest to lowest):
|
* State Priority (highest to lowest):
|
||||||
* 1. Restarting → degraded:unhealthy
|
* 1. Degraded (from sub-resources) → degraded:unhealthy
|
||||||
* 2. Crash Loop (exited with restarts) → degraded:unhealthy
|
* 2. Restarting → degraded:unhealthy
|
||||||
* 3. Mixed (running + exited) → degraded:unhealthy
|
* 3. Crash Loop (exited with restarts) → degraded:unhealthy
|
||||||
* 4. Running → running:healthy/unhealthy/unknown
|
* 4. Mixed (running + exited) → degraded:unhealthy
|
||||||
* 5. Dead/Removing → degraded:unhealthy
|
* 5. Mixed (running + starting) → starting:unknown
|
||||||
* 6. Paused → paused:unknown
|
* 6. Running → running:healthy/unhealthy/unknown
|
||||||
* 7. Starting/Created → starting:unknown
|
* 7. Dead/Removing → degraded:unhealthy
|
||||||
* 8. Exited → exited
|
* 8. Paused → paused:unknown
|
||||||
|
* 9. Starting/Created → starting:unknown
|
||||||
|
* 10. Exited → exited
|
||||||
*/
|
*/
|
||||||
class ContainerStatusAggregator
|
class ContainerStatusAggregator
|
||||||
{
|
{
|
||||||
@@ -64,10 +66,16 @@ class ContainerStatusAggregator
|
|||||||
$hasStarting = false;
|
$hasStarting = false;
|
||||||
$hasPaused = false;
|
$hasPaused = false;
|
||||||
$hasDead = false;
|
$hasDead = false;
|
||||||
|
$hasDegraded = false;
|
||||||
|
|
||||||
// Parse each status string and set flags
|
// Parse each status string and set flags
|
||||||
foreach ($containerStatuses as $status) {
|
foreach ($containerStatuses as $status) {
|
||||||
if (str($status)->contains('restarting')) {
|
if (str($status)->contains('degraded')) {
|
||||||
|
$hasDegraded = true;
|
||||||
|
if (str($status)->contains('unhealthy')) {
|
||||||
|
$hasUnhealthy = true;
|
||||||
|
}
|
||||||
|
} elseif (str($status)->contains('restarting')) {
|
||||||
$hasRestarting = true;
|
$hasRestarting = true;
|
||||||
} elseif (str($status)->contains('running')) {
|
} elseif (str($status)->contains('running')) {
|
||||||
$hasRunning = true;
|
$hasRunning = true;
|
||||||
@@ -98,6 +106,7 @@ class ContainerStatusAggregator
|
|||||||
$hasStarting,
|
$hasStarting,
|
||||||
$hasPaused,
|
$hasPaused,
|
||||||
$hasDead,
|
$hasDead,
|
||||||
|
$hasDegraded,
|
||||||
$maxRestartCount
|
$maxRestartCount
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -175,6 +184,7 @@ class ContainerStatusAggregator
|
|||||||
$hasStarting,
|
$hasStarting,
|
||||||
$hasPaused,
|
$hasPaused,
|
||||||
$hasDead,
|
$hasDead,
|
||||||
|
false, // $hasDegraded - not applicable for container objects, only for status strings
|
||||||
$maxRestartCount
|
$maxRestartCount
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -190,6 +200,7 @@ class ContainerStatusAggregator
|
|||||||
* @param bool $hasStarting Has at least one starting/created container
|
* @param bool $hasStarting Has at least one starting/created container
|
||||||
* @param bool $hasPaused Has at least one paused container
|
* @param bool $hasPaused Has at least one paused container
|
||||||
* @param bool $hasDead Has at least one dead/removing container
|
* @param bool $hasDead Has at least one dead/removing container
|
||||||
|
* @param bool $hasDegraded Has at least one degraded container
|
||||||
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
|
* @param int $maxRestartCount Maximum restart count (for crash loop detection)
|
||||||
* @return string Status in colon format (e.g., "running:healthy")
|
* @return string Status in colon format (e.g., "running:healthy")
|
||||||
*/
|
*/
|
||||||
@@ -202,24 +213,37 @@ class ContainerStatusAggregator
|
|||||||
bool $hasStarting,
|
bool $hasStarting,
|
||||||
bool $hasPaused,
|
bool $hasPaused,
|
||||||
bool $hasDead,
|
bool $hasDead,
|
||||||
|
bool $hasDegraded,
|
||||||
int $maxRestartCount
|
int $maxRestartCount
|
||||||
): string {
|
): string {
|
||||||
// Priority 1: Restarting containers (degraded state)
|
// Priority 1: Degraded containers from sub-resources (highest priority)
|
||||||
|
// If any service/application within a service stack is degraded, the entire stack is degraded
|
||||||
|
if ($hasDegraded) {
|
||||||
|
return 'degraded:unhealthy';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 2: Restarting containers (degraded state)
|
||||||
if ($hasRestarting) {
|
if ($hasRestarting) {
|
||||||
return 'degraded:unhealthy';
|
return 'degraded:unhealthy';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 2: Crash loop detection (exited with restart count > 0)
|
// Priority 3: Crash loop detection (exited with restart count > 0)
|
||||||
if ($hasExited && $maxRestartCount > 0) {
|
if ($hasExited && $maxRestartCount > 0) {
|
||||||
return 'degraded:unhealthy';
|
return 'degraded:unhealthy';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 3: Mixed state (some running, some exited = degraded)
|
// Priority 4: Mixed state (some running, some exited = degraded)
|
||||||
if ($hasRunning && $hasExited) {
|
if ($hasRunning && $hasExited) {
|
||||||
return 'degraded:unhealthy';
|
return 'degraded:unhealthy';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 4: Running containers (check health status)
|
// Priority 5: Mixed state (some running, some starting = still starting)
|
||||||
|
// If any component is still starting, the entire service stack is not fully ready
|
||||||
|
if ($hasRunning && $hasStarting) {
|
||||||
|
return 'starting:unknown';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority 6: Running containers (check health status)
|
||||||
if ($hasRunning) {
|
if ($hasRunning) {
|
||||||
if ($hasUnhealthy) {
|
if ($hasUnhealthy) {
|
||||||
return 'running:unhealthy';
|
return 'running:unhealthy';
|
||||||
@@ -230,22 +254,22 @@ class ContainerStatusAggregator
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 5: Dead or removing containers
|
// Priority 7: Dead or removing containers
|
||||||
if ($hasDead) {
|
if ($hasDead) {
|
||||||
return 'degraded:unhealthy';
|
return 'degraded:unhealthy';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 6: Paused containers
|
// Priority 8: Paused containers
|
||||||
if ($hasPaused) {
|
if ($hasPaused) {
|
||||||
return 'paused:unknown';
|
return 'paused:unknown';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 7: Starting/created containers
|
// Priority 9: Starting/created containers
|
||||||
if ($hasStarting) {
|
if ($hasStarting) {
|
||||||
return 'starting:unknown';
|
return 'starting:unknown';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Priority 8: All containers exited (no restart count = truly stopped)
|
// Priority 10: All containers exited (no restart count = truly stopped)
|
||||||
return 'exited';
|
return 'exited';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -126,6 +126,70 @@ describe('aggregateFromStrings', function () {
|
|||||||
expect($result)->toBe('starting:unknown');
|
expect($result)->toBe('starting:unknown');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('returns degraded:unhealthy for single degraded container', function () {
|
||||||
|
$statuses = collect(['degraded:unhealthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns degraded:unhealthy when mixing degraded with running healthy', function () {
|
||||||
|
$statuses = collect(['degraded:unhealthy', 'running:healthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns degraded:unhealthy when mixing running healthy with degraded', function () {
|
||||||
|
$statuses = collect(['running:healthy', 'degraded:unhealthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns degraded:unhealthy for multiple degraded containers', function () {
|
||||||
|
$statuses = collect(['degraded:unhealthy', 'degraded:unhealthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('degraded status overrides all other non-critical states', function () {
|
||||||
|
$statuses = collect(['degraded:unhealthy', 'running:healthy', 'starting', 'paused']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns starting:unknown when mixing starting with running healthy (service not fully ready)', function () {
|
||||||
|
$statuses = collect(['starting:unknown', 'running:healthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('starting:unknown');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns starting:unknown when mixing created with running healthy', function () {
|
||||||
|
$statuses = collect(['created', 'running:healthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('starting:unknown');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('returns starting:unknown for multiple starting containers with some running', function () {
|
||||||
|
$statuses = collect(['starting:unknown', 'starting:unknown', 'running:healthy']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('starting:unknown');
|
||||||
|
});
|
||||||
|
|
||||||
test('handles parentheses format input (backward compatibility)', function () {
|
test('handles parentheses format input (backward compatibility)', function () {
|
||||||
$statuses = collect(['running (healthy)', 'running (unhealthy)']);
|
$statuses = collect(['running (healthy)', 'running (unhealthy)']);
|
||||||
|
|
||||||
@@ -166,8 +230,16 @@ describe('aggregateFromStrings', function () {
|
|||||||
expect($result)->toBe('degraded:unhealthy');
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('prioritizes running over paused/starting/exited', function () {
|
test('mixed running and starting returns starting', function () {
|
||||||
$statuses = collect(['running:healthy', 'starting', 'paused']);
|
$statuses = collect(['running:healthy', 'starting']);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('starting:unknown');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('prioritizes running over paused/exited when no starting', function () {
|
||||||
|
$statuses = collect(['running:healthy', 'paused', 'exited']);
|
||||||
|
|
||||||
$result = $this->aggregator->aggregateFromStrings($statuses);
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
@@ -398,7 +470,23 @@ describe('aggregateFromContainers', function () {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe('state priority enforcement', function () {
|
describe('state priority enforcement', function () {
|
||||||
test('restarting has highest priority', function () {
|
test('degraded from sub-resources has highest priority', function () {
|
||||||
|
$statuses = collect([
|
||||||
|
'degraded:unhealthy',
|
||||||
|
'restarting',
|
||||||
|
'running:healthy',
|
||||||
|
'dead',
|
||||||
|
'paused',
|
||||||
|
'starting',
|
||||||
|
'exited',
|
||||||
|
]);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('restarting has second highest priority', function () {
|
||||||
$statuses = collect([
|
$statuses = collect([
|
||||||
'restarting',
|
'restarting',
|
||||||
'running:healthy',
|
'running:healthy',
|
||||||
@@ -413,7 +501,7 @@ describe('state priority enforcement', function () {
|
|||||||
expect($result)->toBe('degraded:unhealthy');
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('crash loop has second highest priority', function () {
|
test('crash loop has third highest priority', function () {
|
||||||
$statuses = collect([
|
$statuses = collect([
|
||||||
'exited',
|
'exited',
|
||||||
'running:healthy',
|
'running:healthy',
|
||||||
@@ -426,7 +514,7 @@ describe('state priority enforcement', function () {
|
|||||||
expect($result)->toBe('degraded:unhealthy');
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('mixed state (running + exited) has third priority', function () {
|
test('mixed state (running + exited) has fourth priority', function () {
|
||||||
$statuses = collect([
|
$statuses = collect([
|
||||||
'running:healthy',
|
'running:healthy',
|
||||||
'exited',
|
'exited',
|
||||||
@@ -439,6 +527,18 @@ describe('state priority enforcement', function () {
|
|||||||
expect($result)->toBe('degraded:unhealthy');
|
expect($result)->toBe('degraded:unhealthy');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('mixed state (running + starting) has fifth priority', function () {
|
||||||
|
$statuses = collect([
|
||||||
|
'running:healthy',
|
||||||
|
'starting',
|
||||||
|
'paused',
|
||||||
|
]);
|
||||||
|
|
||||||
|
$result = $this->aggregator->aggregateFromStrings($statuses);
|
||||||
|
|
||||||
|
expect($result)->toBe('starting:unknown');
|
||||||
|
});
|
||||||
|
|
||||||
test('running:unhealthy has priority over running:unknown', function () {
|
test('running:unhealthy has priority over running:unknown', function () {
|
||||||
$statuses = collect([
|
$statuses = collect([
|
||||||
'running:unknown',
|
'running:unknown',
|
||||||
|
|||||||
Reference in New Issue
Block a user