fix(sentinel): reduce resource churn from health flaps

Ignore health status changes in Sentinel push deduplication when the container lifecycle state is unchanged. Scope stale resource checks to Sentinel servers whose heartbeat is stale, and avoid refreshing resource last_online_at on unchanged statuses.
2026-09-10 06:20:26 +00:00 · 2026-05-27 16:48:38 +02:00
parent 1c5d5676ef
commit 626cfb4a22
8 changed files with 410 additions and 70 deletions
@@ -3,9 +3,11 @@
 namespace App\Actions\Server;

 use App\Models\Application;
+use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
 use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
 use App\Models\StandaloneDragonfly;
 use App\Models\StandaloneKeydb;
 use App\Models\StandaloneMariadb;
@@ -13,6 +15,8 @@ use App\Models\StandaloneMongodb;
 use App\Models\StandaloneMysql;
 use App\Models\StandalonePostgresql;
 use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
+use Illuminate\Support\Collection;
 use Lorisleiva\Actions\Concerns\AsAction;

 class ResourcesCheck
@@ -21,21 +25,73 @@ class ResourcesCheck

    public function handle()
    {
-        $seconds = 60;
+        $seconds = config('constants.sentinel.resource_stale_seconds', 300);
+        $staleServerIds = $this->staleSentinelServerIds($seconds);
+
+        if ($staleServerIds->isEmpty()) {
+            return;
+        }
+
+        [$standaloneDockerIds, $swarmDockerIds] = $this->destinationIdsForServers($staleServerIds);
+
        try {
-            Application::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceApplication::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            ServiceDatabase::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandalonePostgresql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneRedis::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMongodb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMysql::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneMariadb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneKeydb::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneDragonfly::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
-            StandaloneClickhouse::where('last_online_at', '<', now()->subSeconds($seconds))->update(['status' => 'exited']);
+            Application::where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            ServiceApplication::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            ServiceDatabase::whereHas('service', fn ($query) => $query->whereIn('server_id', $staleServerIds))
+                ->where('status', 'not like', 'exited%')
+                ->update(['status' => 'exited']);
+
+            collect([
+                StandalonePostgresql::class,
+                StandaloneRedis::class,
+                StandaloneMongodb::class,
+                StandaloneMysql::class,
+                StandaloneMariadb::class,
+                StandaloneKeydb::class,
+                StandaloneDragonfly::class,
+                StandaloneClickhouse::class,
+            ])->each(function (string $databaseClass) use ($standaloneDockerIds, $swarmDockerIds) {
+                $databaseClass::query()
+                    ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                    ->where('status', 'not like', 'exited%')
+                    ->update(['status' => 'exited']);
+            });
        } catch (\Throwable $e) {
            return handleError($e);
        }
    }
+
+    private function staleSentinelServerIds(int $seconds): Collection
+    {
+        return Server::query()
+            ->whereNotNull('sentinel_updated_at')
+            ->where('sentinel_updated_at', '<', now()->subSeconds($seconds))
+            ->whereHas('settings', fn ($query) => $query->where('is_sentinel_enabled', true))
+            ->pluck('id');
+    }
+
+    private function destinationIdsForServers(Collection $serverIds): array
+    {
+        return [
+            StandaloneDocker::whereIn('server_id', $serverIds)->pluck('id'),
+            SwarmDocker::whereIn('server_id', $serverIds)->pluck('id'),
+        ];
+    }
+
+    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+    {
+        $query->where(function ($query) use ($standaloneDockerIds) {
+            $query->where('destination_type', StandaloneDocker::class)
+                ->whereIn('destination_id', $standaloneDockerIds);
+        })->orWhere(function ($query) use ($swarmDockerIds) {
+            $query->where('destination_type', SwarmDocker::class)
+                ->whereIn('destination_id', $swarmDockerIds);
+        });
+    }
 }
@@ -143,11 +143,13 @@ class SentinelController extends Controller
    /**
     * Build a stable hash of container state.
     *
-     * Covers [name, state, health_status] only — metrics and
-     * filesystem_usage_root are excluded on purpose (disk % churns constantly
-     * and would defeat the hash; the storage check is separately cache-gated
-     * inside PushServerUpdateJob). Sorted by name so container ordering from
-     * Sentinel does not affect the hash.
+     * Covers [name, state] only — metrics, filesystem_usage_root, and
+     * health_status are excluded on purpose. Disk % churns constantly, and
+     * health checks can flap between starting/healthy/unhealthy while the
+     * container lifecycle state remains unchanged. Both would otherwise defeat
+     * the hash and dispatch DB-heavy PushServerUpdateJob instances too often.
+     * The force window still refreshes full state periodically. Sorted by name
+     * so container ordering from Sentinel does not affect the hash.
     */
    private function containerStateHash(array $data): string
    {
@@ -155,7 +157,6 @@ class SentinelController extends Controller
            ->map(fn ($c) => [
                'name' => data_get($c, 'name'),
                'state' => data_get($c, 'state'),
-                'health_status' => data_get($c, 'health_status'),
            ])
            ->sortBy('name')
            ->values()
@@ -13,6 +13,16 @@ use App\Models\ApplicationPreview;
 use App\Models\Server;
 use App\Models\ServiceApplication;
 use App\Models\ServiceDatabase;
+use App\Models\StandaloneClickhouse;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneDragonfly;
+use App\Models\StandaloneKeydb;
+use App\Models\StandaloneMariadb;
+use App\Models\StandaloneMongodb;
+use App\Models\StandaloneMysql;
+use App\Models\StandalonePostgresql;
+use App\Models\StandaloneRedis;
+use App\Models\SwarmDocker;
 use App\Notifications\Container\ContainerRestarted;
 use App\Services\ContainerStatusAggregator;
 use App\Traits\CalculatesExcludedStatus;
@@ -25,6 +35,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\DB;
 use Laravel\Horizon\Contracts\Silenced;

 class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -46,6 +57,18 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    public Collection $services;

+    public Collection $applicationsById;
+
+    public Collection $previewsByKey;
+
+    public Collection $databasesByUuid;
+
+    public Collection $servicesById;
+
+    public Collection $serviceApplicationsById;
+
+    public Collection $serviceDatabasesById;
+
    public Collection $allApplicationIds;

    public Collection $allDatabaseUuids;
@@ -103,6 +126,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->allTcpProxyUuids = collect();
        $this->allServiceApplicationIds = collect();
        $this->allServiceDatabaseIds = collect();
+        $this->applicationsById = collect();
+        $this->previewsByKey = collect();
+        $this->databasesByUuid = collect();
+        $this->servicesById = collect();
+        $this->serviceApplicationsById = collect();
+        $this->serviceDatabasesById = collect();
    }

    public function handle()
@@ -120,6 +149,12 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->allTcpProxyUuids ??= collect();
        $this->allServiceApplicationIds ??= collect();
        $this->allServiceDatabaseIds ??= collect();
+        $this->applicationsById ??= collect();
+        $this->previewsByKey ??= collect();
+        $this->databasesByUuid ??= collect();
+        $this->servicesById ??= collect();
+        $this->serviceApplicationsById ??= collect();
+        $this->serviceDatabasesById ??= collect();

        // TODO: Swarm is not supported yet
        if (! $this->data) {
@@ -151,13 +186,16 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            return;
        }

-        $this->applications = $this->server->applications();
-        $this->databases = $this->server->databases();
-        $this->previews = $this->server->previews();
-        // Eager load service applications and databases to avoid N+1 queries
-        $this->services = $this->server->services()
-            ->with(['applications:id,service_id', 'databases:id,service_id'])
-            ->get();
+        $this->applications = $this->loadApplications();
+        $this->databases = $this->loadDatabases();
+        $this->previews = $this->loadPreviews();
+        $this->services = $this->loadServices();
+        $this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
+        $this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
+        $this->databasesByUuid = $this->databases->keyBy('uuid');
+        $this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
+        $this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
+        $this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);

        $this->allApplicationIds = $this->applications->filter(function ($application) {
            return $application->additional_servers_count === 0;
@@ -170,9 +208,8 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        });
        $this->allDatabaseUuids = $this->databases->pluck('uuid');
        $this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
-        // Use eager-loaded relationships instead of querying in loop
-        $this->allServiceApplicationIds = $this->services->flatMap(fn ($service) => $service->applications->pluck('id'));
-        $this->allServiceDatabaseIds = $this->services->flatMap(fn ($service) => $service->databases->pluck('id'));
+        $this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
+        $this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();

        foreach ($this->containers as $container) {
            $containerStatus = data_get($container, 'state', 'exited');
@@ -286,6 +323,142 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        $this->checkLogDrainContainer();
    }

+    private function loadApplications(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+
+        $applications = Application::withoutGlobalScope('withRelations')
+            ->select([
+                'id',
+                'uuid',
+                'name',
+                'status',
+                'build_pack',
+                'docker_compose_raw',
+                'destination_id',
+                'destination_type',
+                'last_online_at',
+            ])
+            ->withCount('additional_servers')
+            ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+            ->get();
+
+        $additionalApplicationIds = DB::table('additional_destinations')
+            ->where('server_id', $this->server->id)
+            ->pluck('application_id');
+
+        if ($additionalApplicationIds->isNotEmpty()) {
+            $applications = $applications->concat(
+                Application::withoutGlobalScope('withRelations')
+                    ->select([
+                        'id',
+                        'uuid',
+                        'name',
+                        'status',
+                        'build_pack',
+                        'docker_compose_raw',
+                        'destination_id',
+                        'destination_type',
+                        'last_online_at',
+                    ])
+                    ->withCount('additional_servers')
+                    ->whereIn('id', $additionalApplicationIds)
+                    ->get()
+            );
+        }
+
+        return $applications->unique('id')->values();
+    }
+
+    private function loadPreviews(): Collection
+    {
+        $applicationIds = $this->applications->pluck('id');
+
+        if ($applicationIds->isEmpty()) {
+            return collect();
+        }
+
+        return ApplicationPreview::query()
+            ->select([
+                'id',
+                'application_id',
+                'pull_request_id',
+                'status',
+                'last_online_at',
+            ])
+            ->whereIn('application_id', $applicationIds)
+            ->get();
+    }
+
+    private function loadServices(): Collection
+    {
+        return $this->server->services()
+            ->select([
+                'id',
+                'server_id',
+                'uuid',
+                'docker_compose_raw',
+            ])
+            ->with([
+                'applications:id,service_id,status,last_online_at',
+                'databases:id,service_id,status,last_online_at,is_public,name',
+            ])
+            ->get();
+    }
+
+    private function loadDatabases(): Collection
+    {
+        [$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
+        $databaseColumns = [
+            'id',
+            'uuid',
+            'name',
+            'status',
+            'is_public',
+            'destination_id',
+            'destination_type',
+            'last_online_at',
+            'restart_count',
+            'last_restart_at',
+            'last_restart_type',
+        ];
+
+        return collect([
+            StandalonePostgresql::class,
+            StandaloneRedis::class,
+            StandaloneMongodb::class,
+            StandaloneMysql::class,
+            StandaloneMariadb::class,
+            StandaloneKeydb::class,
+            StandaloneDragonfly::class,
+            StandaloneClickhouse::class,
+        ])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
+            return $databaseClass::query()
+                ->select($databaseColumns)
+                ->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
+                ->get();
+        })->filter(fn ($database) => data_get($database, 'name') !== 'coolify-db')->values();
+    }
+
+    private function serverDestinationIds(): array
+    {
+        return [
+            StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
+            SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
+        ];
+    }
+
+    private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
+    {
+        $query->where(function ($query) use ($standaloneDockerIds) {
+            $query->where('destination_type', StandaloneDocker::class)
+                ->whereIn('destination_id', $standaloneDockerIds);
+        })->orWhere(function ($query) use ($swarmDockerIds) {
+            $query->where('destination_type', SwarmDocker::class)
+                ->whereIn('destination_id', $swarmDockerIds);
+        });
+    }
+
    private function aggregateMultiContainerStatuses()
    {
        if ($this->applicationContainerStatuses->isEmpty()) {
@@ -293,7 +466,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        }

        foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
-            $application = $this->applications->where('id', $applicationId)->first();
+            $application = $this->applicationsById->get((string) $applicationId);
            if (! $application) {
                continue;
            }
@@ -314,8 +487,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                    $application->status = $aggregatedStatus;
                    $application->save();
-                } elseif ($aggregatedStatus) {
-                    $application->update(['last_online_at' => now()]);
                }

                continue;
@@ -330,8 +501,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
                $application->status = $aggregatedStatus;
                $application->save();
-            } elseif ($aggregatedStatus) {
-                $application->update(['last_online_at' => now()]);
            }
        }
    }
@@ -350,7 +519,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                continue;
            }

-            $service = $this->services->where('id', $serviceId)->first();
+            $service = $this->servicesById->get((string) $serviceId);
            if (! $service) {
                continue;
            }
@@ -358,9 +527,9 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            // Get the service sub-resource (ServiceApplication or ServiceDatabase)
            $subResource = null;
            if ($subType === 'application') {
-                $subResource = $service->applications->where('id', $subId)->first();
+                $subResource = $this->serviceApplicationsById->get((string) $subId);
            } elseif ($subType === 'database') {
-                $subResource = $service->databases->where('id', $subId)->first();
+                $subResource = $this->serviceDatabasesById->get((string) $subId);
            }

            if (! $subResource) {
@@ -382,8 +551,6 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
                if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                    $subResource->status = $aggregatedStatus;
                    $subResource->save();
-                } elseif ($aggregatedStatus) {
-                    $subResource->update(['last_online_at' => now()]);
                }

                continue;
@@ -399,39 +566,31 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
                $subResource->status = $aggregatedStatus;
                $subResource->save();
-            } elseif ($aggregatedStatus) {
-                $subResource->update(['last_online_at' => now()]);
            }
        }
    }

    private function updateApplicationStatus(string $applicationId, string $containerStatus)
    {
-        $application = $this->applications->where('id', $applicationId)->first();
+        $application = $this->applicationsById->get((string) $applicationId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
        }
    }

    private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
    {
-        $application = $this->previews->where('application_id', $applicationId)
-            ->where('pull_request_id', $pullRequestId)
-            ->first();
+        $application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
        if (! $application) {
            return;
        }
        if ($application->status !== $containerStatus) {
            $application->status = $containerStatus;
            $application->save();
-        } else {
-            $application->update(['last_online_at' => now()]);
        }
    }

@@ -479,9 +638,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
            $applicationId = $parts[0];
            $pullRequestId = $parts[1];

-            $applicationPreview = $this->previews->where('application_id', $applicationId)
-                ->where('pull_request_id', $pullRequestId)
-                ->first();
+            $applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);

            if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
                $previewIdsToUpdate->push($applicationPreview->id);
@@ -520,15 +677,13 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced

    private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, bool $tcpProxy = false)
    {
-        $database = $this->databases->where('uuid', $databaseUuid)->first();
+        $database = $this->databasesByUuid->get($databaseUuid);
        if (! $database) {
            return;
        }
        if ($database->status !== $containerStatus) {
            $database->status = $containerStatus;
            $database->save();
-        } else {
-            $database->update(['last_online_at' => now()]);
        }
        if ($this->isRunning($containerStatus) && $tcpProxy) {
            $tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
@@ -563,7 +718,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
        }

        $notFoundDatabaseUuids->each(function ($databaseUuid) {
-            $database = $this->databases->where('uuid', $databaseUuid)->first();
+            $database = $this->databasesByUuid->get($databaseUuid);
            if ($database) {
                if (! str($database->status)->startsWith('exited')) {
                    $database->update([
@@ -93,9 +93,15 @@ return [

    'sentinel' => [
        // How often (seconds) PushServerUpdateJob is force-dispatched even when
-        // the container state hash is unchanged. Keeps last_online_at,
-        // exited-detection and storage checks from going stale.
+        // the container state hash is unchanged. Keeps exited-detection and
+        // storage checks from going stale without writing every resource row on
+        // every push.
        'push_force_interval_seconds' => env('SENTINEL_PUSH_FORCE_INTERVAL_SECONDS', 300),
+
+        // How long a Sentinel-enabled server may go without a heartbeat before
+        // ResourcesCheck considers its resources stale. Per-resource
+        // last_online_at is only updated on real status changes, not every push.
+        'resource_stale_seconds' => env('SENTINEL_RESOURCE_STALE_SECONDS', 300),
    ],

    'proxy' => [
@@ -1,17 +1,19 @@
 <?php

 use App\Jobs\PushServerUpdateJob;
+use App\Models\Environment;
+use App\Models\Project;
 use App\Models\Server;
+use App\Models\StandaloneDocker;
 use App\Models\StandalonePostgresql;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;

 uses(RefreshDatabase::class);

-test('database last_online_at is updated when status unchanged', function () {
+test('database last_online_at is not updated when status is unchanged', function () {
    $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
        'status' => 'running:healthy',
        'last_online_at' => now()->subMinutes(5),
    ]);
@@ -40,15 +42,13 @@ test('database last_online_at is updated when status unchanged', function () {

    $database->refresh();

-    // last_online_at should be updated even though status didn't change
-    expect($database->last_online_at->greaterThan($oldLastOnline))->toBeTrue();
+    expect((string) $database->last_online_at)->toBe((string) $oldLastOnline);
    expect($database->status)->toBe('running:healthy');
 });

 test('database status is updated when container status changes', function () {
    $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
        'status' => 'exited',
    ]);

@@ -79,8 +79,7 @@ test('database status is updated when container status changes', function () {

 test('database is not marked exited when containers list is empty', function () {
    $team = Team::factory()->create();
-    $database = StandalonePostgresql::factory()->create([
-        'team_id' => $team->id,
+    $database = createPushUpdatePostgresql($team, [
        'status' => 'running:healthy',
    ]);

@@ -99,3 +98,31 @@ test('database is not marked exited when containers list is empty', function ()
    // Status should remain running, NOT be set to exited
    expect($database->status)->toBe('running:healthy');
 });
+
+function createPushUpdatePostgresql(Team $team, array $attributes = []): StandalonePostgresql
+{
+    $lastOnlineAt = $attributes['last_online_at'] ?? null;
+    unset($attributes['last_online_at']);
+
+    $server = Server::factory()->create(['team_id' => $team->id]);
+    $destination = StandaloneDocker::where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    $database = StandalonePostgresql::create(array_merge([
+        'uuid' => (string) str()->uuid(),
+        'name' => 'postgres-'.str()->random(8),
+        'postgres_password' => 'secret',
+        'status' => 'exited',
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+        'environment_id' => $environment->id,
+    ], $attributes));
+
+    if ($lastOnlineAt !== null) {
+        $database->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+    }
+
+    return $database;
+}
@@ -4,17 +4,21 @@ use App\Jobs\PushServerUpdateJob;
 use App\Models\Server;
 use App\Models\Service;
 use App\Models\ServiceApplication;
+use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;

 uses(RefreshDatabase::class);

 test('containers with empty service subId are skipped', function () {
-    $server = Server::factory()->create();
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
    $service = Service::factory()->create([
        'server_id' => $server->id,
    ]);
-    $serviceApp = ServiceApplication::factory()->create([
+    $serviceApp = ServiceApplication::create([
        'service_id' => $service->id,
+        'uuid' => (string) str()->uuid(),
+        'name' => 'app-'.str()->random(8),
    ]);

    $data = [
@@ -44,12 +48,15 @@ test('containers with empty service subId are skipped', function () {
 });

 test('containers with valid service subId are processed', function () {
-    $server = Server::factory()->create();
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(['team_id' => $team->id]);
    $service = Service::factory()->create([
        'server_id' => $server->id,
    ]);
-    $serviceApp = ServiceApplication::factory()->create([
+    $serviceApp = ServiceApplication::create([
        'service_id' => $service->id,
+        'uuid' => (string) str()->uuid(),
+        'name' => 'app-'.str()->random(8),
    ]);

    $data = [
@@ -0,0 +1,78 @@
+<?php
+
+use App\Actions\Server\ResourcesCheck;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+
+uses(RefreshDatabase::class);
+
+it('does not mark resources exited when sentinel is still reporting for the server', function () {
+    Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+    config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+    $application = resourcesCheckApplication([
+        'last_online_at' => now()->subHour(),
+        'status' => 'running:healthy',
+    ], [
+        'sentinel_updated_at' => now()->subMinute(),
+    ]);
+
+    ResourcesCheck::run();
+
+    $application->refresh();
+
+    expect($application->status)->toBe('running:healthy');
+});
+
+it('marks resources exited when their sentinel server is stale', function () {
+    Carbon::setTestNow(Carbon::create(2026, 5, 27, 12, 0, 0, 'UTC'));
+    config(['constants.sentinel.resource_stale_seconds' => 300]);
+
+    $application = resourcesCheckApplication([
+        'last_online_at' => now()->subHour(),
+        'status' => 'running:healthy',
+    ], [
+        'sentinel_updated_at' => now()->subMinutes(10),
+    ]);
+
+    ResourcesCheck::run();
+
+    $application->refresh();
+
+    expect($application->status)->toBe('exited:unhealthy');
+});
+
+function resourcesCheckApplication(array $applicationAttributes = [], array $serverAttributes = []): Application
+{
+    $lastOnlineAt = $applicationAttributes['last_online_at'] ?? null;
+    unset($applicationAttributes['last_online_at']);
+
+    $team = Team::factory()->create();
+    $server = Server::factory()->create(array_merge([
+        'team_id' => $team->id,
+    ], $serverAttributes));
+    $server->settings()->update(['is_sentinel_enabled' => true]);
+
+    $destination = StandaloneDocker::where('server_id', $server->id)->first()
+        ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);
+    $project = Project::factory()->create(['team_id' => $team->id]);
+    $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+    $application = Application::factory()->create(array_merge([
+        'environment_id' => $environment->id,
+        'destination_id' => $destination->id,
+        'destination_type' => $destination->getMorphClass(),
+    ], $applicationAttributes));
+
+    if ($lastOnlineAt !== null) {
+        $application->forceFill(['last_online_at' => $lastOnlineAt])->saveQuietly();
+    }
+
+    return $application;
+}
@@ -138,6 +138,16 @@ it('dispatches the job when container state changes', function () use ($running)
    Queue::assertPushed(PushServerUpdateJob::class, 2);
 });

+it('ignores health status changes while container lifecycle state is unchanged', function () {
+    $healthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'healthy']];
+    $unhealthy = [['name' => 'app-1', 'state' => 'running', 'health_status' => 'unhealthy']];
+
+    pushSentinel($this->token, sentinelPayload($healthy))->assertOk();
+    pushSentinel($this->token, sentinelPayload($unhealthy))->assertOk();
+
+    Queue::assertPushed(PushServerUpdateJob::class, 1);
+});
+
 it('ignores disk percentage changes (excluded from the hash)', function () use ($running) {
    pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 42.0))->assertOk();
    pushSentinel($this->token, sentinelPayload($running(), diskPercentage: 88.0))->assertOk();