Fix complex status logic: handle degraded sub-resources and mixed running+starting states

- Add support for degraded status from sub-resources as highest priority
- Handle mixed running+starting state to show service as not fully ready
- Update state priority hierarchy from 8 to 10 levels
- Add comprehensive test coverage for new status scenarios

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
2025-12-02 21:47:15 +01:00
parent afb19114cf
commit c65ad2e655
2 changed files with 146 additions and 22 deletions

View File

@@ -16,14 +16,16 @@ use Illuminate\Support\Facades\Log;
* UI components transform this to human-readable format (e.g., "Running (Healthy)"). * UI components transform this to human-readable format (e.g., "Running (Healthy)").
* *
* State Priority (highest to lowest): * State Priority (highest to lowest):
* 1. Restarting degraded:unhealthy * 1. Degraded (from sub-resources) degraded:unhealthy
* 2. Crash Loop (exited with restarts) degraded:unhealthy * 2. Restarting degraded:unhealthy
* 3. Mixed (running + exited) degraded:unhealthy * 3. Crash Loop (exited with restarts) degraded:unhealthy
* 4. Running running:healthy/unhealthy/unknown * 4. Mixed (running + exited) degraded:unhealthy
* 5. Dead/Removing degraded:unhealthy * 5. Mixed (running + starting) starting:unknown
* 6. Paused paused:unknown * 6. Running running:healthy/unhealthy/unknown
* 7. Starting/Created starting:unknown * 7. Dead/Removing degraded:unhealthy
* 8. Exited exited * 8. Paused paused:unknown
* 9. Starting/Created starting:unknown
* 10. Exited exited
*/ */
class ContainerStatusAggregator class ContainerStatusAggregator
{ {
@@ -64,10 +66,16 @@ class ContainerStatusAggregator
$hasStarting = false; $hasStarting = false;
$hasPaused = false; $hasPaused = false;
$hasDead = false; $hasDead = false;
$hasDegraded = false;
// Parse each status string and set flags // Parse each status string and set flags
foreach ($containerStatuses as $status) { foreach ($containerStatuses as $status) {
if (str($status)->contains('restarting')) { if (str($status)->contains('degraded')) {
$hasDegraded = true;
if (str($status)->contains('unhealthy')) {
$hasUnhealthy = true;
}
} elseif (str($status)->contains('restarting')) {
$hasRestarting = true; $hasRestarting = true;
} elseif (str($status)->contains('running')) { } elseif (str($status)->contains('running')) {
$hasRunning = true; $hasRunning = true;
@@ -98,6 +106,7 @@ class ContainerStatusAggregator
$hasStarting, $hasStarting,
$hasPaused, $hasPaused,
$hasDead, $hasDead,
$hasDegraded,
$maxRestartCount $maxRestartCount
); );
} }
@@ -175,6 +184,7 @@ class ContainerStatusAggregator
$hasStarting, $hasStarting,
$hasPaused, $hasPaused,
$hasDead, $hasDead,
false, // $hasDegraded - not applicable for container objects, only for status strings
$maxRestartCount $maxRestartCount
); );
} }
@@ -190,6 +200,7 @@ class ContainerStatusAggregator
* @param bool $hasStarting Has at least one starting/created container * @param bool $hasStarting Has at least one starting/created container
* @param bool $hasPaused Has at least one paused container * @param bool $hasPaused Has at least one paused container
* @param bool $hasDead Has at least one dead/removing container * @param bool $hasDead Has at least one dead/removing container
* @param bool $hasDegraded Has at least one degraded container
* @param int $maxRestartCount Maximum restart count (for crash loop detection) * @param int $maxRestartCount Maximum restart count (for crash loop detection)
* @return string Status in colon format (e.g., "running:healthy") * @return string Status in colon format (e.g., "running:healthy")
*/ */
@@ -202,24 +213,37 @@ class ContainerStatusAggregator
bool $hasStarting, bool $hasStarting,
bool $hasPaused, bool $hasPaused,
bool $hasDead, bool $hasDead,
bool $hasDegraded,
int $maxRestartCount int $maxRestartCount
): string { ): string {
// Priority 1: Restarting containers (degraded state) // Priority 1: Degraded containers from sub-resources (highest priority)
// If any service/application within a service stack is degraded, the entire stack is degraded
if ($hasDegraded) {
return 'degraded:unhealthy';
}
// Priority 2: Restarting containers (degraded state)
if ($hasRestarting) { if ($hasRestarting) {
return 'degraded:unhealthy'; return 'degraded:unhealthy';
} }
// Priority 2: Crash loop detection (exited with restart count > 0) // Priority 3: Crash loop detection (exited with restart count > 0)
if ($hasExited && $maxRestartCount > 0) { if ($hasExited && $maxRestartCount > 0) {
return 'degraded:unhealthy'; return 'degraded:unhealthy';
} }
// Priority 3: Mixed state (some running, some exited = degraded) // Priority 4: Mixed state (some running, some exited = degraded)
if ($hasRunning && $hasExited) { if ($hasRunning && $hasExited) {
return 'degraded:unhealthy'; return 'degraded:unhealthy';
} }
// Priority 4: Running containers (check health status) // Priority 5: Mixed state (some running, some starting = still starting)
// If any component is still starting, the entire service stack is not fully ready
if ($hasRunning && $hasStarting) {
return 'starting:unknown';
}
// Priority 6: Running containers (check health status)
if ($hasRunning) { if ($hasRunning) {
if ($hasUnhealthy) { if ($hasUnhealthy) {
return 'running:unhealthy'; return 'running:unhealthy';
@@ -230,22 +254,22 @@ class ContainerStatusAggregator
} }
} }
// Priority 5: Dead or removing containers // Priority 7: Dead or removing containers
if ($hasDead) { if ($hasDead) {
return 'degraded:unhealthy'; return 'degraded:unhealthy';
} }
// Priority 6: Paused containers // Priority 8: Paused containers
if ($hasPaused) { if ($hasPaused) {
return 'paused:unknown'; return 'paused:unknown';
} }
// Priority 7: Starting/created containers // Priority 9: Starting/created containers
if ($hasStarting) { if ($hasStarting) {
return 'starting:unknown'; return 'starting:unknown';
} }
// Priority 8: All containers exited (no restart count = truly stopped) // Priority 10: All containers exited (no restart count = truly stopped)
return 'exited'; return 'exited';
} }
} }

View File

@@ -126,6 +126,70 @@ describe('aggregateFromStrings', function () {
expect($result)->toBe('starting:unknown'); expect($result)->toBe('starting:unknown');
}); });
test('returns degraded:unhealthy for single degraded container', function () {
$statuses = collect(['degraded:unhealthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('returns degraded:unhealthy when mixing degraded with running healthy', function () {
$statuses = collect(['degraded:unhealthy', 'running:healthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('returns degraded:unhealthy when mixing running healthy with degraded', function () {
$statuses = collect(['running:healthy', 'degraded:unhealthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('returns degraded:unhealthy for multiple degraded containers', function () {
$statuses = collect(['degraded:unhealthy', 'degraded:unhealthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('degraded status overrides all other non-critical states', function () {
$statuses = collect(['degraded:unhealthy', 'running:healthy', 'starting', 'paused']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('returns starting:unknown when mixing starting with running healthy (service not fully ready)', function () {
$statuses = collect(['starting:unknown', 'running:healthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('starting:unknown');
});
test('returns starting:unknown when mixing created with running healthy', function () {
$statuses = collect(['created', 'running:healthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('starting:unknown');
});
test('returns starting:unknown for multiple starting containers with some running', function () {
$statuses = collect(['starting:unknown', 'starting:unknown', 'running:healthy']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('starting:unknown');
});
test('handles parentheses format input (backward compatibility)', function () { test('handles parentheses format input (backward compatibility)', function () {
$statuses = collect(['running (healthy)', 'running (unhealthy)']); $statuses = collect(['running (healthy)', 'running (unhealthy)']);
@@ -166,8 +230,16 @@ describe('aggregateFromStrings', function () {
expect($result)->toBe('degraded:unhealthy'); expect($result)->toBe('degraded:unhealthy');
}); });
test('prioritizes running over paused/starting/exited', function () { test('mixed running and starting returns starting', function () {
$statuses = collect(['running:healthy', 'starting', 'paused']); $statuses = collect(['running:healthy', 'starting']);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('starting:unknown');
});
test('prioritizes running over paused/exited when no starting', function () {
$statuses = collect(['running:healthy', 'paused', 'exited']);
$result = $this->aggregator->aggregateFromStrings($statuses); $result = $this->aggregator->aggregateFromStrings($statuses);
@@ -398,7 +470,23 @@ describe('aggregateFromContainers', function () {
}); });
describe('state priority enforcement', function () { describe('state priority enforcement', function () {
test('restarting has highest priority', function () { test('degraded from sub-resources has highest priority', function () {
$statuses = collect([
'degraded:unhealthy',
'restarting',
'running:healthy',
'dead',
'paused',
'starting',
'exited',
]);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('degraded:unhealthy');
});
test('restarting has second highest priority', function () {
$statuses = collect([ $statuses = collect([
'restarting', 'restarting',
'running:healthy', 'running:healthy',
@@ -413,7 +501,7 @@ describe('state priority enforcement', function () {
expect($result)->toBe('degraded:unhealthy'); expect($result)->toBe('degraded:unhealthy');
}); });
test('crash loop has second highest priority', function () { test('crash loop has third highest priority', function () {
$statuses = collect([ $statuses = collect([
'exited', 'exited',
'running:healthy', 'running:healthy',
@@ -426,7 +514,7 @@ describe('state priority enforcement', function () {
expect($result)->toBe('degraded:unhealthy'); expect($result)->toBe('degraded:unhealthy');
}); });
test('mixed state (running + exited) has third priority', function () { test('mixed state (running + exited) has fourth priority', function () {
$statuses = collect([ $statuses = collect([
'running:healthy', 'running:healthy',
'exited', 'exited',
@@ -439,6 +527,18 @@ describe('state priority enforcement', function () {
expect($result)->toBe('degraded:unhealthy'); expect($result)->toBe('degraded:unhealthy');
}); });
test('mixed state (running + starting) has fifth priority', function () {
$statuses = collect([
'running:healthy',
'starting',
'paused',
]);
$result = $this->aggregator->aggregateFromStrings($statuses);
expect($result)->toBe('starting:unknown');
});
test('running:unhealthy has priority over running:unknown', function () { test('running:unhealthy has priority over running:unknown', function () {
$statuses = collect([ $statuses = collect([
'running:unknown', 'running:unknown',